Merge pull request #64 from scrapinghub/fix_unit_tests

Fix unit tests by setting botgroup's max_running
scrapinghub · Apr 4, 2016 · 7857954 · 7857954
2 parents 31525d3 + afaada9
commit 7857954
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 14 deletions.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,32 @@
+.PHONY: release release-minor release-path release-major upload test test-all lint clean
+
+VERSION?=minor
+
+release:
+	bumpversion $(VERSION)
+
+release-minor: release
+
+release-patch:
+	make release VERSION=patch
+
+release-major:
+	make release VERSION=major
+
+upload: clean
+	python setup.py sdist bdist_wheel upload
+
+test:
+	py.test tests/
+
+test-all:
+	tox
+
+lint:
+	flake8 hubstorage
+
+clean:
+	rm -rf build/ dist/ *.egg-info htmlcov/
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
diff --git a/tests/hstestcase.py b/tests/hstestcase.py
@@ -1,7 +1,9 @@
 import os
 import unittest
 import random
+import requests
 from hubstorage import HubstorageClient
+from hubstorage.utils import urlpathjoin
 
 
 class HSTestCase(unittest.TestCase):
@@ -12,7 +14,7 @@ class HSTestCase(unittest.TestCase):
     auth = os.getenv('HS_AUTH', 'f' * 32)
     frontier = 'test'
     slot = 'site.com'
-    testbotgroup = 'python-hubstorage-test'
+    testbotgroups = ['python-hubstorage-test', 'g1']
 
     @classmethod
     def setUpClass(cls):
@@ -63,13 +65,22 @@ def _delete_job(cls, jobkey):
 
     @classmethod
     def _set_testbotgroup(cls):
-        cls.project.settings.apipost(jl={'botgroups': [cls.testbotgroup]})
+        cls.project.settings.apipost(jl={'botgroups': [cls.testbotgroups[0]]})
+        # Additional step to populate JobQ's botgroups table
+        for botgroup in cls.testbotgroups:
+            url = urlpathjoin(cls.endpoint, 'botgroups',
+                              botgroup, 'max_running')
+            requests.post(url, auth=cls.project.auth, data='null')
         cls.project.settings.expire()
 
     @classmethod
     def _unset_testbotgroup(cls):
         cls.project.settings.apidelete('botgroups')
         cls.project.settings.expire()
+        # Additional step to delete botgroups in JobQ
+        for botgroup in cls.testbotgroups:
+            url = urlpathjoin(cls.endpoint, 'botgroups', botgroup)
+            requests.delete(url, auth=cls.project.auth)
 
     def start_job(self, **startparams):
         jobdata = self.project.jobq.start(**startparams)

diff --git a/tests/test_jobq.py b/tests/test_jobq.py
@@ -142,7 +142,7 @@ def test_summary_jobmeta(self):
 
     def test_summary_countstart(self):
         # push more than 5 jobs into same queue
-        N = 20
+        N = 6
         jobq = self.project.jobq
         for state in ('pending', 'running', 'finished'):
             for idx in xrange(N):

diff --git a/tests/test_retry.py b/tests/test_retry.py
@@ -42,7 +42,7 @@ def test_delete_on_hubstorage_api_does_not_404(self):
     @responses.activate
     def test_retrier_does_not_catch_unwanted_exception(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback, attempts_count = self.make_request_callback(3, job_metadata, http_error_status=403)
 
@@ -65,7 +65,7 @@ def test_retrier_does_not_catch_unwanted_exception(self):
     @responses.activate
     def test_retrier_catches_badstatusline_and_429(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
 
         attempts_count = [0]  # use a list for nonlocal mutability used in request_callback
@@ -93,7 +93,7 @@ def request_callback(request):
     @responses.activate
     def test_api_delete_can_be_set_to_non_idempotent(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback_delete, attempts_count_delete = self.make_request_callback(2, job_metadata)
 
@@ -115,7 +115,7 @@ def test_api_delete_can_be_set_to_non_idempotent(self):
     @responses.activate
     def test_collection_store_and_delete_are_retried(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
 
         callback_post, attempts_count_post = self.make_request_callback(2, [])
         callback_delete, attempts_count_delete = self.make_request_callback(2, [])
@@ -136,7 +136,7 @@ def test_collection_store_and_delete_are_retried(self):
     @responses.activate
     def test_delete_requests_are_retried(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback_getpost, attempts_count_getpost = self.make_request_callback(0, job_metadata)
         callback_delete, attempts_count_delete = self.make_request_callback(2, job_metadata)
@@ -157,7 +157,7 @@ def test_delete_requests_are_retried(self):
     @responses.activate
     def test_metadata_save_does_retry(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback_get, attempts_count_get = self.make_request_callback(0, job_metadata)
         callback_post, attempts_count_post = self.make_request_callback(2, job_metadata)
@@ -197,7 +197,7 @@ def test_push_job_does_not_retry(self):
     @responses.activate
     def test_get_job_does_retry(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=3, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback, attempts_count = self.make_request_callback(2, job_metadata)
 
@@ -236,7 +236,7 @@ def test_get_job_does_fails_if_no_retries(self):
     @responses.activate
     def test_get_job_does_fails_on_too_many_retries(self):
         # Prepare
-        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2)
+        client = HubstorageClient(auth=self.auth, endpoint=self.endpoint, max_retries=2, max_retry_time=1)
         job_metadata = {'project': self.projectid, 'spider': self.spidername, 'state': 'pending'}
         callback, attempts_count = self.make_request_callback(3, job_metadata)
 

diff --git a/tox.ini b/tox.ini
@@ -4,12 +4,18 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, pypy, py3
+envlist = py27, pypy
 
 [testenv]
 deps =
     -rrequirements.txt
     responses==0.5.0
-    nose
+    pytest
+    pytest-cov
 
-commands = nosetests []
+commands = py.test --cov {posargs}
+
+addopts =
+    -v  # verbose mode
+    -r a  # show extra test summary info
+    --doctest-modules  # run doctests in all .py modules