From 63505a423bf0062e98cf1b97407df78a1a693f18 Mon Sep 17 00:00:00 2001 From: Colin Nattrass Date: Wed, 26 Oct 2022 17:55:31 +0200 Subject: [PATCH 1/6] Switch Docker base image from CentOS (as it is now EOL) (close #283) PR #284 * Switch Docker base image to Debian (closes snowplow#283) * Update git clone depth --- Dockerfile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d0577e8..ff25c7bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,16 @@ -FROM centos:8 -RUN cd /etc/yum.repos.d/ -RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* -RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y --no-install-recommends make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + mecab-ipadic-utf8 git ca-certificates -RUN yum -y install wget -RUN yum install -y epel-release -RUN yum -y install git tar gcc make bzip2 openssl openssl-devel patch gcc-c++ libffi-devel sqlite-devel -RUN git clone https://github.com/yyuu/pyenv.git ~/.pyenv ENV HOME /root ENV PYENV_ROOT $HOME/.pyenv ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +RUN git clone --depth=1 https://github.com/pyenv/pyenv.git $PYENV_ROOT +RUN git clone --depth=1 https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv RUN pyenv install 3.5.10 && pyenv install 3.6.14 && pyenv install 3.7.11 && pyenv install 3.8.11 && pyenv install 3.9.6 && pyenv install 3.10.1 -RUN git clone https://github.com/pyenv/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv WORKDIR /app COPY . . From 6e886b7aa46074e5536a6169af43f37340a748cc Mon Sep 17 00:00:00 2001 From: Colin Nattrass Date: Wed, 26 Oct 2022 17:57:26 +0200 Subject: [PATCH 2/6] Add session id and idx to Subject (closes #282) PR #285 * Add domain_session_id and domain_session_index to Subject Class --- snowplow_tracker/subject.py | 20 +++++++++++++++++++ .../test/integration/test_integration.py | 6 +++++- snowplow_tracker/test/unit/test_subject.py | 8 ++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/snowplow_tracker/subject.py b/snowplow_tracker/subject.py index d9c10c80..3b4fb82c 100644 --- a/snowplow_tracker/subject.py +++ b/snowplow_tracker/subject.py @@ -123,6 +123,26 @@ def set_domain_user_id(self, duid: str) -> 'Subject': self.standard_nv_pairs["duid"] = duid return self + def set_domain_session_id(self, sid: str) -> 'Subject': + """ + Set the domain session ID + :param sid: Domain session ID + :type sid: string + :rtype: subject + """ + self.standard_nv_pairs["sid"] = sid + return self + + def set_domain_session_index(self, vid: int) -> 'Subject': + """ + Set the domain session Index + :param vid: Domain session Index + :type vid: int + :rtype: subject + """ + self.standard_nv_pairs["vid"] = vid + return self + def set_ip_address(self, ip: str) -> 'Subject': """ Set the domain user ID diff --git a/snowplow_tracker/test/integration/test_integration.py b/snowplow_tracker/test/integration/test_integration.py index cea52a46..2346243b 100644 --- a/snowplow_tracker/test/integration/test_integration.py +++ b/snowplow_tracker/test/integration/test_integration.py @@ -226,6 +226,8 @@ def test_integration_standard_nv_pairs(self) -> None: def test_integration_identification_methods(self) -> None: s = subject.Subject() s.set_domain_user_id("4616bfb38f872d16") + s.set_domain_session_id("59ed13b1a5724dae") + s.set_domain_session_index(1) s.set_ip_address("255.255.255.255") s.set_useragent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)") s.set_network_user_id("fbc6c76c-bce5-43ce-8d5a-31c5") @@ -235,6 +237,8 @@ def test_integration_identification_methods(self) -> None: t.track_page_view("localhost", "local host") expected_fields = { "duid": "4616bfb38f872d16", + "sid": "59ed13b1a5724dae", + "vid": "1", "ip": "255.255.255.255", "ua": "Mozilla%2F5.0+%28compatible%3B+MSIE+9.0%3B+Windows+NT+6.0%3B+Trident%2F5.0%29", "tnuid": "fbc6c76c-bce5-43ce-8d5a-31c5" @@ -245,7 +249,7 @@ def test_integration_identification_methods(self) -> None: def test_integration_event_subject(self) -> None: s = subject.Subject() s.set_domain_user_id("4616bfb38f872d16") - s.set_ip_address("255.255.255.255") + s.set_lang("ES") t = tracker.Tracker([emitters.Emitter("localhost")], s, "cf", app_id="angry-birds-android") evSubject = subject.Subject().set_domain_user_id("1111aaa11a111a11").set_lang("EN") diff --git a/snowplow_tracker/test/unit/test_subject.py b/snowplow_tracker/test/unit/test_subject.py index 882cdb2f..93e2b278 100644 --- a/snowplow_tracker/test/unit/test_subject.py +++ b/snowplow_tracker/test/unit/test_subject.py @@ -42,6 +42,8 @@ def test_subject_0(self) -> None: s.set_timezone("PST") s.set_lang("EN") s.set_domain_user_id("domain-user-id") + s.set_domain_session_id("domain-session-id") + s.set_domain_session_index(1) s.set_ip_address("127.0.0.1") s.set_useragent("useragent-string") s.set_network_user_id("network-user-id") @@ -57,6 +59,8 @@ def test_subject_0(self) -> None: "ip": "127.0.0.1", "ua": "useragent-string", "duid": "domain-user-id", + "sid": "domain-session-id", + "vid": 1, "tnuid": "network-user-id" } self.assertDictEqual(s.standard_nv_pairs, exp) @@ -85,5 +89,9 @@ def test_subject_1(self) -> None: s.standard_nv_pairs["ua"] with pytest.raises(KeyError): s.standard_nv_pairs["duid"] + with pytest.raises(KeyError): + s.standard_nv_pairs["sid"] + with pytest.raises(KeyError): + s.standard_nv_pairs["vid"] with pytest.raises(KeyError): s.standard_nv_pairs["tnuid"] From 7feeee905bcd180ba66da71125b944b9df4337ff Mon Sep 17 00:00:00 2001 From: Jack-Keene <87364579+Jack-Keene@users.noreply.github.com> Date: Mon, 31 Oct 2022 12:38:35 +0000 Subject: [PATCH 3/6] Add Python 3.11 to CI tests (close #286) PR #287 * Add Python 3.11 to CI tests --- .github/workflows/ci.yml | 2 +- Dockerfile | 2 +- run-tests.sh | 28 ++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f92176df..e5d5fb09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] extras-required: [".", ".[redis]"] services: diff --git a/Dockerfile b/Dockerfile index ff25c7bd..c48e4a8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH RUN git clone --depth=1 https://github.com/pyenv/pyenv.git $PYENV_ROOT RUN git clone --depth=1 https://github.com/pyenv/pyenv-virtualenv.git $PYENV_ROOT/plugins/pyenv-virtualenv -RUN pyenv install 3.5.10 && pyenv install 3.6.14 && pyenv install 3.7.11 && pyenv install 3.8.11 && pyenv install 3.9.6 && pyenv install 3.10.1 +RUN pyenv install 3.5.10 && pyenv install 3.6.14 && pyenv install 3.7.11 && pyenv install 3.8.11 && pyenv install 3.9.6 && pyenv install 3.10.1 && pyenv install 3.11.0 WORKDIR /app COPY . . diff --git a/run-tests.sh b/run-tests.sh index 477f3f7e..715f72dd 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -116,6 +116,23 @@ function deploy { pip install -r requirements-test.txt source deactivate fi + + # pyenv install 3.11.0 + if [ ! -e ~/.pyenv/versions/tracker311 ]; then + pyenv virtualenv 3.11.0 tracker311 + pyenv activate tracker311 + pip install . + pip install -r requirements-test.txt + source deactivate + fi + + if [ ! -e ~/.pyenv/versions/tracker311redis ]; then + pyenv virtualenv 3.11.0 tracker311redis + pyenv activate tracker311redis + pip install .[redis] + pip install -r requirements-test.txt + source deactivate + fi } @@ -167,6 +184,15 @@ function run_tests { pyenv activate tracker310redis pytest source deactivate + + pyenv activate tracker311 + pytest + source deactivate + + pyenv activate tracker311redis + pytest + source deactivate + } function refresh_deploy { @@ -182,6 +208,8 @@ function refresh_deploy { pyenv uninstall -f tracker39redis pyenv uninstall -f tracker310 pyenv uninstall -f tracker310redis + pyenv uninstall -f tracker311 + pyenv uninstall -f tracker311redis } diff --git a/setup.py b/setup.py index 6dd3bc73..6f359ceb 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: OS Independent", ], install_requires=["requests>=2.25.1,<3.0", "typing_extensions>=3.7.4"], From fed98f8f61cae0329efaddf46eb262705c9bafd2 Mon Sep 17 00:00:00 2001 From: Jack-Keene <87364579+Jack-Keene@users.noreply.github.com> Date: Mon, 31 Oct 2022 16:01:16 +0000 Subject: [PATCH 4/6] Make HTTPS the default protocol in emitter (close #14) PR #288 * Set https as default protocol * Add unit tests Make HTTPS the default protocol in emitter (close #14) #288 * Fix endpoint check --- snowplow_tracker/emitters.py | 13 ++++++++---- snowplow_tracker/test/unit/test_emitters.py | 22 ++++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/snowplow_tracker/emitters.py b/snowplow_tracker/emitters.py index 2deb0345..0f331ef5 100644 --- a/snowplow_tracker/emitters.py +++ b/snowplow_tracker/emitters.py @@ -51,7 +51,7 @@ class Emitter(object): def __init__( self, endpoint: str, - protocol: HttpProtocol = "http", + protocol: HttpProtocol = "https", port: Optional[int] = None, method: Method = "get", buffer_size: Optional[int] = None, @@ -60,9 +60,9 @@ def __init__( byte_limit: Optional[int] = None, request_timeout: Optional[Union[float, Tuple[float, float]]] = None) -> None: """ - :param endpoint: The collector URL. Don't include "http://" - this is done automatically. + :param endpoint: The collector URL. If protocol is not set in endpoint it will automatically set to "https://" - this is done automatically. :type endpoint: string - :param protocol: The protocol to use - http or https. Defaults to http. + :param protocol: The protocol to use - http or https. Defaults to https. :type protocol: protocol :param port: The collector port to connect to :type port: int | None @@ -116,7 +116,7 @@ def __init__( @staticmethod def as_collector_uri( endpoint: str, - protocol: HttpProtocol = "http", + protocol: HttpProtocol = "https", port: Optional[int] = None, method: Method = "get") -> str: """ @@ -133,6 +133,11 @@ def as_collector_uri( if len(endpoint) < 1: raise ValueError("No endpoint provided.") + if endpoint.split("://")[0] in PROTOCOLS: + endpoint_arr = endpoint.split("://") + protocol = endpoint_arr[0] + endpoint = endpoint_arr[1] + if method == "get": path = "/i" else: diff --git a/snowplow_tracker/test/unit/test_emitters.py b/snowplow_tracker/test/unit/test_emitters.py index 0167525a..00ff68c1 100644 --- a/snowplow_tracker/test/unit/test_emitters.py +++ b/snowplow_tracker/test/unit/test_emitters.py @@ -54,7 +54,7 @@ def setUp(self) -> None: def test_init(self) -> None: e = Emitter('0.0.0.0') - self.assertEqual(e.endpoint, 'http://0.0.0.0/i') + self.assertEqual(e.endpoint, 'https://0.0.0.0/i') self.assertEqual(e.method, 'get') self.assertEqual(e.buffer_size, 1) self.assertEqual(e.buffer, []) @@ -83,24 +83,32 @@ def test_init_requests_timeout(self) -> None: def test_as_collector_uri(self) -> None: uri = Emitter.as_collector_uri('0.0.0.0') - self.assertEqual(uri, 'http://0.0.0.0/i') + self.assertEqual(uri, 'https://0.0.0.0/i') def test_as_collector_uri_post(self) -> None: uri = Emitter.as_collector_uri('0.0.0.0', method="post") - self.assertEqual(uri, 'http://0.0.0.0/com.snowplowanalytics.snowplow/tp2') + self.assertEqual(uri, 'https://0.0.0.0/com.snowplowanalytics.snowplow/tp2') def test_as_collector_uri_port(self) -> None: uri = Emitter.as_collector_uri('0.0.0.0', port=9090, method="post") - self.assertEqual(uri, 'http://0.0.0.0:9090/com.snowplowanalytics.snowplow/tp2') + self.assertEqual(uri, 'https://0.0.0.0:9090/com.snowplowanalytics.snowplow/tp2') - def test_as_collector_uri_https(self) -> None: - uri = Emitter.as_collector_uri('0.0.0.0', protocol="https") - self.assertEqual(uri, 'https://0.0.0.0/i') + def test_as_collector_uri_http(self) -> None: + uri = Emitter.as_collector_uri('0.0.0.0', protocol="http") + self.assertEqual(uri, 'http://0.0.0.0/i') def test_as_collector_uri_empty_string(self) -> None: with self.assertRaises(ValueError): Emitter.as_collector_uri('') + def test_as_collector_uri_endpoint_protocol(self) -> None: + uri = Emitter.as_collector_uri("https://0.0.0.0") + self.assertEqual(uri, "https://0.0.0.0/i") + + def test_as_collector_uri_endpoint_protocol_http(self) -> None: + uri = Emitter.as_collector_uri("http://0.0.0.0") + self.assertEqual(uri, "http://0.0.0.0/i") + @mock.patch('snowplow_tracker.Emitter.flush') def test_input_no_flush(self, mok_flush: Any) -> None: mok_flush.side_effect = mocked_flush From 2cd6abbbc7b9554d8bbcfca0eeee5b91875e42ca Mon Sep 17 00:00:00 2001 From: Jack-Keene <87364579+Jack-Keene@users.noreply.github.com> Date: Thu, 3 Nov 2022 10:06:04 +0000 Subject: [PATCH 5/6] Change default method to POST in emitter (close #289) PR #290 * Set default method to post * Fix invalid escape sequence in doc strings * Update integration tests * Update unit tests * Update example app --- examples/app.py | 3 +- snowplow_tracker/celery/celery_emitter.py | 2 +- snowplow_tracker/emitters.py | 16 +++---- snowplow_tracker/payload.py | 4 +- .../test/integration/test_integration.py | 48 ++++++++++--------- snowplow_tracker/test/unit/test_emitters.py | 40 ++++++++-------- snowplow_tracker/tracker.py | 8 ++-- 7 files changed, 62 insertions(+), 59 deletions(-) diff --git a/examples/app.py b/examples/app.py index 829055d0..973f5a99 100644 --- a/examples/app.py +++ b/examples/app.py @@ -19,7 +19,7 @@ def main(): t = Tracker(e, s) - print("Sending events to " + collector_url) + print("Sending events to " + e.endpoint) t.track_page_view("https://www.snowplow.io", "Homepage") t.track_page_ping("https://www.snowplow.io", "Homepage") @@ -32,6 +32,7 @@ def main(): ) ) t.track_struct_event("shop", "add-to-basket", None, "pcs", 2) + t.flush() if __name__ == "__main__": diff --git a/snowplow_tracker/celery/celery_emitter.py b/snowplow_tracker/celery/celery_emitter.py index e7a8efae..d9aafaa7 100644 --- a/snowplow_tracker/celery/celery_emitter.py +++ b/snowplow_tracker/celery/celery_emitter.py @@ -52,7 +52,7 @@ def __init__( endpoint: str, protocol: HttpProtocol = "http", port: Optional[int] = None, - method: Method = "get", + method: Method = "post", buffer_size: Optional[int] = None, byte_limit: Optional[int] = None) -> None: super(CeleryEmitter, self).__init__(endpoint, protocol, port, method, buffer_size, None, None, byte_limit) diff --git a/snowplow_tracker/emitters.py b/snowplow_tracker/emitters.py index 0f331ef5..d2549d25 100644 --- a/snowplow_tracker/emitters.py +++ b/snowplow_tracker/emitters.py @@ -53,7 +53,7 @@ def __init__( endpoint: str, protocol: HttpProtocol = "https", port: Optional[int] = None, - method: Method = "get", + method: Method = "post", buffer_size: Optional[int] = None, on_success: Optional[SuccessCallback] = None, on_failure: Optional[FailureCallback] = None, @@ -66,7 +66,7 @@ def __init__( :type protocol: protocol :param port: The collector port to connect to :type port: int | None - :param method: The HTTP request method + :param method: The HTTP request method. Defaults to post. :type method: method :param buffer_size: The maximum number of queued events before the buffer is flushed. Default is 10. :type buffer_size: int | None @@ -118,7 +118,7 @@ def as_collector_uri( endpoint: str, protocol: HttpProtocol = "https", port: Optional[int] = None, - method: Method = "get") -> str: + method: Method = "post") -> str: """ :param endpoint: The raw endpoint provided by the user :type endpoint: string @@ -153,7 +153,7 @@ def input(self, payload: PayloadDict) -> None: If the maximum size has been reached, flushes the buffer. :param payload: The name-value pairs for the event - :type payload: dict(string:\*) + :type payload: dict(string:\\*) """ with self.lock: if self.bytes_queued is not None: @@ -212,7 +212,7 @@ def http_post(self, data: str) -> bool: def http_get(self, payload: PayloadDict) -> bool: """ :param payload: The event properties - :type payload: dict(string:\*) + :type payload: dict(string:\\*) """ logger.info("Sending GET request to %s..." % self.endpoint) logger.debug("Payload: %s" % payload) @@ -247,7 +247,7 @@ def is_good_status_code(status_code: int) -> bool: def send_events(self, evts: PayloadDictList) -> None: """ :param evts: Array of events to be sent - :type evts: list(dict(string:\*)) + :type evts: list(dict(string:\\*)) """ if len(evts) > 0: logger.info("Attempting to send %s events" % len(evts)) @@ -312,7 +312,7 @@ def attach_sent_timestamp(events: PayloadDictList) -> None: as `stm` param :param events: Array of events to be sent - :type events: list(dict(string:\*)) + :type events: list(dict(string:\\*)) :rtype: None """ def update(e: PayloadDict) -> None: @@ -332,7 +332,7 @@ def __init__( endpoint: str, protocol: HttpProtocol = "http", port: Optional[int] = None, - method: Method = "get", + method: Method = "post", buffer_size: Optional[int] = None, on_success: Optional[SuccessCallback] = None, on_failure: Optional[FailureCallback] = None, diff --git a/snowplow_tracker/payload.py b/snowplow_tracker/payload.py index 77fa6759..bb47a1d6 100644 --- a/snowplow_tracker/payload.py +++ b/snowplow_tracker/payload.py @@ -54,7 +54,7 @@ def add_dict(self, dict_: PayloadDict, base64: bool = False) -> None: Add a dict of name value pairs to the Payload object :param dict_: Dictionary to be added to the Payload - :type dict_: dict(string:\*) + :type dict_: dict(string:\\*) """ for f in dict_: self.add(f, dict_[f]) @@ -70,7 +70,7 @@ def add_json( Add an encoded or unencoded JSON to the payload :param dict_: Custom context for the event - :type dict_: dict(string:\*) | None + :type dict_: dict(string:\\*) | None :param encode_base64: If the payload is base64 encoded :type encode_base64: bool :param type_when_encoded: Name of the field when encode_base64 is set diff --git a/snowplow_tracker/test/integration/test_integration.py b/snowplow_tracker/test/integration/test_integration.py index 2346243b..c487bb70 100644 --- a/snowplow_tracker/test/integration/test_integration.py +++ b/snowplow_tracker/test/integration/test_integration.py @@ -36,9 +36,9 @@ querystrings = [""] -default_emitter = emitters.Emitter("localhost", protocol="http", port=80) +default_emitter = emitters.Emitter("localhost", protocol="http", port=80, buffer_size=1) -post_emitter = emitters.Emitter("localhost", protocol="http", port=80, method='post', buffer_size=1) +get_emitter = emitters.Emitter("localhost", protocol="http", port=80, method='get') default_subject = subject.Subject() @@ -79,7 +79,7 @@ def fail_response_content(url: str, request: Any) -> Dict[str, Any]: class IntegrationTest(unittest.TestCase): def test_integration_page_view(self) -> None: - t = tracker.Tracker([default_emitter], default_subject) + t = tracker.Tracker([get_emitter], default_subject) with HTTMock(pass_response_content): t.track_page_view("http://savethearctic.org", "Save The Arctic", "http://referrer.com") expected_fields = {"e": "pv", "page": "Save+The+Arctic", "url": "http%3A%2F%2Fsavethearctic.org", "refr": "http%3A%2F%2Freferrer.com"} @@ -87,7 +87,7 @@ def test_integration_page_view(self) -> None: self.assertEqual(from_querystring(key, querystrings[-1]), expected_fields[key]) def test_integration_ecommerce_transaction_item(self) -> None: - t = tracker.Tracker([default_emitter], default_subject) + t = tracker.Tracker([get_emitter], default_subject) with HTTMock(pass_response_content): t.track_ecommerce_transaction_item("12345", "pbz0025", 7.99, 2, "black-tarot", "tarot", currency="GBP") expected_fields = {"ti_ca": "tarot", "ti_id": "12345", "ti_qu": "2", "ti_sk": "pbz0025", "e": "ti", "ti_nm": "black-tarot", "ti_pr": "7.99", "ti_cu": "GBP"} @@ -95,7 +95,7 @@ def test_integration_ecommerce_transaction_item(self) -> None: self.assertEqual(from_querystring(key, querystrings[-1]), expected_fields[key]) def test_integration_ecommerce_transaction(self) -> None: - t = tracker.Tracker([default_emitter], default_subject) + t = tracker.Tracker([get_emitter], default_subject) with HTTMock(pass_response_content): t.track_ecommerce_transaction( "6a8078be", 35, city="London", currency="GBP", @@ -126,7 +126,7 @@ def test_integration_ecommerce_transaction(self) -> None: self.assertEqual(from_querystring("ttm", querystrings[-3]), from_querystring("ttm", querystrings[-2])) def test_integration_screen_view(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=False) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=False) with HTTMock(pass_response_content): t.track_screen_view("Game HUD 2", id_="534") expected_fields = {"e": "ue"} @@ -146,7 +146,7 @@ def test_integration_screen_view(self) -> None: }) def test_integration_struct_event(self) -> None: - t = tracker.Tracker([default_emitter], default_subject) + t = tracker.Tracker([get_emitter], default_subject) with HTTMock(pass_response_content): t.track_struct_event("Ecomm", "add-to-basket", "dog-skateboarding-video", "hd", 13.99) expected_fields = {"se_ca": "Ecomm", "se_pr": "hd", "se_la": "dog-skateboarding-video", "se_va": "13.99", "se_ac": "add-to-basket", "e": "se"} @@ -154,7 +154,7 @@ def test_integration_struct_event(self) -> None: self.assertEqual(from_querystring(key, querystrings[-1]), expected_fields[key]) def test_integration_unstruct_event_non_base64(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=False) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=False) with HTTMock(pass_response_content): t.track_unstruct_event(SelfDescribingJson("iglu:com.acme/viewed_product/jsonschema/2-0-2", {"product_id": "ASO01043", "price$flt": 49.95, "walrus$tms": 1000})) expected_fields = {"e": "ue"} @@ -168,7 +168,7 @@ def test_integration_unstruct_event_non_base64(self) -> None: }) def test_integration_unstruct_event_base64(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=True) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=True) with HTTMock(pass_response_content): t.track_unstruct_event(SelfDescribingJson("iglu:com.acme/viewed_product/jsonschema/2-0-2", {"product_id": "ASO01043", "price$flt": 49.95, "walrus$tms": 1000})) expected_fields = {"e": "ue"} @@ -182,7 +182,7 @@ def test_integration_unstruct_event_base64(self) -> None: }) def test_integration_context_non_base64(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=False) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=False) with HTTMock(pass_response_content): t.track_page_view("localhost", "local host", None, [SelfDescribingJson("iglu:com.example/user/jsonschema/2-0-3", {"user_type": "tester"})]) envelope_string = from_querystring("co", querystrings[-1]) @@ -193,7 +193,7 @@ def test_integration_context_non_base64(self) -> None: }) def test_integration_context_base64(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=True) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=True) with HTTMock(pass_response_content): t.track_page_view("localhost", "local host", None, [SelfDescribingJson("iglu:com.example/user/jsonschema/2-0-3", {"user_type": "tester"})]) envelope_string = unquote_plus(from_querystring("cx", querystrings[-1])) @@ -212,7 +212,7 @@ def test_integration_standard_nv_pairs(self) -> None: s.set_timezone("Europe London") s.set_lang("en") - t = tracker.Tracker([emitters.Emitter("localhost")], s, "cf", app_id="angry-birds-android") + t = tracker.Tracker([emitters.Emitter("localhost", method='get')], s, "cf", app_id="angry-birds-android") with HTTMock(pass_response_content): t.track_page_view("localhost", "local host") expected_fields = {"tna": "cf", "res": "100x200", @@ -232,7 +232,7 @@ def test_integration_identification_methods(self) -> None: s.set_useragent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)") s.set_network_user_id("fbc6c76c-bce5-43ce-8d5a-31c5") - t = tracker.Tracker([emitters.Emitter("localhost")], s, "cf", app_id="angry-birds-android") + t = tracker.Tracker([emitters.Emitter("localhost", method='get')], s, "cf", app_id="angry-birds-android") with HTTMock(pass_response_content): t.track_page_view("localhost", "local host") expected_fields = { @@ -251,7 +251,7 @@ def test_integration_event_subject(self) -> None: s.set_domain_user_id("4616bfb38f872d16") s.set_lang("ES") - t = tracker.Tracker([emitters.Emitter("localhost")], s, "cf", app_id="angry-birds-android") + t = tracker.Tracker([emitters.Emitter("localhost", method='get')], s, "cf", app_id="angry-birds-android") evSubject = subject.Subject().set_domain_user_id("1111aaa11a111a11").set_lang("EN") with HTTMock(pass_response_content): t.track_page_view("localhost", "local host", event_subject=evSubject) @@ -293,6 +293,7 @@ def test_integration_success_callback(self) -> None: callback_failure_queue = [] callback_emitter = emitters.Emitter( "localhost", + method='get', on_success=lambda x: callback_success_queue.append(x), on_failure=lambda x, y: callback_failure_queue.append(x)) t = tracker.Tracker([callback_emitter], default_subject) @@ -312,6 +313,7 @@ def test_integration_failure_callback(self) -> None: callback_failure_queue = [] callback_emitter = emitters.Emitter( "localhost", + method='get', on_success=lambda x: callback_success_queue.append(x), on_failure=lambda x, y: callback_failure_queue.append(x)) t = tracker.Tracker([callback_emitter], default_subject) @@ -321,7 +323,7 @@ def test_integration_failure_callback(self) -> None: self.assertEqual(callback_failure_queue[0], 0) def test_post_page_view(self) -> None: - t = tracker.Tracker([post_emitter], default_subject) + t = tracker.Tracker([default_emitter], default_subject) with HTTMock(pass_post_response_content): t.track_page_view("localhost", "local host", None) expected_fields = {"e": "pv", "page": "local host", "url": "localhost"} @@ -331,8 +333,8 @@ def test_post_page_view(self) -> None: self.assertEqual(request["data"][0][key], expected_fields[key]) def test_post_batched(self) -> None: - post_emitter = emitters.Emitter("localhost", protocol="http", port=80, method='post', buffer_size=2) - t = tracker.Tracker(post_emitter, default_subject) + default_emitter = emitters.Emitter("localhost", protocol="http", port=80, buffer_size=2) + t = tracker.Tracker(default_emitter, default_subject) with HTTMock(pass_post_response_content): t.track_struct_event("Test", "A") t.track_struct_event("Test", "B") @@ -341,7 +343,7 @@ def test_post_batched(self) -> None: @freeze_time("2021-04-19 00:00:01") # unix: 1618790401000 def test_timestamps(self) -> None: - emitter = emitters.Emitter("localhost", protocol="http", port=80, method='post', buffer_size=3) + emitter = emitters.Emitter("localhost", protocol="http", port=80, buffer_size=3) t = tracker.Tracker([emitter], default_subject) with HTTMock(pass_post_response_content): t.track_page_view("localhost", "stamp0", None, tstamp=None) @@ -361,18 +363,18 @@ def test_timestamps(self) -> None: self.assertEqual(request["data"][i].get("stm"), expected_timestamps[i]["stm"]) def test_bytelimit(self) -> None: - post_emitter = emitters.Emitter("localhost", protocol="http", port=80, method='post', buffer_size=5, byte_limit=420) - t = tracker.Tracker(post_emitter, default_subject) + default_emitter = emitters.Emitter("localhost", protocol="http", port=80, buffer_size=5, byte_limit=420) + t = tracker.Tracker(default_emitter, default_subject) with HTTMock(pass_post_response_content): t.track_struct_event("Test", "A") # 140 bytes t.track_struct_event("Test", "A") # 280 bytes t.track_struct_event("Test", "A") # 420 bytes. Send t.track_struct_event("Test", "AA") # 141 self.assertEqual(len(querystrings[-1]["data"]), 3) - self.assertEqual(post_emitter.bytes_queued, 136 + len(_version.__version__)) + self.assertEqual(default_emitter.bytes_queued, 136 + len(_version.__version__)) def test_unicode_get(self) -> None: - t = tracker.Tracker([default_emitter], default_subject, encode_base64=False) + t = tracker.Tracker([get_emitter], default_subject, encode_base64=False) unicode_a = u'\u0107' unicode_b = u'test.\u0107om' test_ctx = SelfDescribingJson('iglu:a.b/c/jsonschema/1-0-0', {'test': unicode_a}) @@ -396,7 +398,7 @@ def test_unicode_get(self) -> None: self.assertEqual(actual_b, unicode_b) def test_unicode_post(self) -> None: - t = tracker.Tracker([post_emitter], default_subject, encode_base64=False) + t = tracker.Tracker([default_emitter], default_subject, encode_base64=False) unicode_a = u'\u0107' unicode_b = u'test.\u0107om' test_ctx = SelfDescribingJson('iglu:a.b/c/jsonschema/1-0-0', {'test': unicode_a}) diff --git a/snowplow_tracker/test/unit/test_emitters.py b/snowplow_tracker/test/unit/test_emitters.py index 00ff68c1..51b107d0 100644 --- a/snowplow_tracker/test/unit/test_emitters.py +++ b/snowplow_tracker/test/unit/test_emitters.py @@ -54,9 +54,9 @@ def setUp(self) -> None: def test_init(self) -> None: e = Emitter('0.0.0.0') - self.assertEqual(e.endpoint, 'https://0.0.0.0/i') - self.assertEqual(e.method, 'get') - self.assertEqual(e.buffer_size, 1) + self.assertEqual(e.endpoint, 'https://0.0.0.0/com.snowplowanalytics.snowplow/tp2') + self.assertEqual(e.method, 'post') + self.assertEqual(e.buffer_size, 10) self.assertEqual(e.buffer, []) self.assertIsNone(e.byte_limit) self.assertIsNone(e.bytes_queued) @@ -70,7 +70,7 @@ def test_init_buffer_size(self) -> None: self.assertEqual(e.buffer_size, 10) def test_init_post(self) -> None: - e = Emitter('0.0.0.0', method="post") + e = Emitter('0.0.0.0') self.assertEqual(e.buffer_size, DEFAULT_MAX_LENGTH) def test_init_byte_limit(self) -> None: @@ -83,19 +83,19 @@ def test_init_requests_timeout(self) -> None: def test_as_collector_uri(self) -> None: uri = Emitter.as_collector_uri('0.0.0.0') - self.assertEqual(uri, 'https://0.0.0.0/i') - - def test_as_collector_uri_post(self) -> None: - uri = Emitter.as_collector_uri('0.0.0.0', method="post") self.assertEqual(uri, 'https://0.0.0.0/com.snowplowanalytics.snowplow/tp2') + def test_as_collector_uri_get(self) -> None: + uri = Emitter.as_collector_uri('0.0.0.0', method='get') + self.assertEqual(uri, 'https://0.0.0.0/i') + def test_as_collector_uri_port(self) -> None: - uri = Emitter.as_collector_uri('0.0.0.0', port=9090, method="post") + uri = Emitter.as_collector_uri('0.0.0.0', port=9090) self.assertEqual(uri, 'https://0.0.0.0:9090/com.snowplowanalytics.snowplow/tp2') def test_as_collector_uri_http(self) -> None: uri = Emitter.as_collector_uri('0.0.0.0', protocol="http") - self.assertEqual(uri, 'http://0.0.0.0/i') + self.assertEqual(uri, 'http://0.0.0.0/com.snowplowanalytics.snowplow/tp2') def test_as_collector_uri_empty_string(self) -> None: with self.assertRaises(ValueError): @@ -103,11 +103,11 @@ def test_as_collector_uri_empty_string(self) -> None: def test_as_collector_uri_endpoint_protocol(self) -> None: uri = Emitter.as_collector_uri("https://0.0.0.0") - self.assertEqual(uri, "https://0.0.0.0/i") + self.assertEqual(uri, "https://0.0.0.0/com.snowplowanalytics.snowplow/tp2") def test_as_collector_uri_endpoint_protocol_http(self) -> None: uri = Emitter.as_collector_uri("http://0.0.0.0") - self.assertEqual(uri, "http://0.0.0.0/i") + self.assertEqual(uri, "http://0.0.0.0/com.snowplowanalytics.snowplow/tp2") @mock.patch('snowplow_tracker.Emitter.flush') def test_input_no_flush(self, mok_flush: Any) -> None: @@ -173,7 +173,7 @@ def test_input_bytes_queued(self, mok_flush: Any) -> None: def test_input_bytes_post(self, mok_flush: Any) -> None: mok_flush.side_effect = mocked_flush - e = Emitter('0.0.0.0', method="post") + e = Emitter('0.0.0.0') nvPairs = {"testString": "test", "testNum": 2.72} e.input(nvPairs) @@ -219,7 +219,7 @@ def test_attach_sent_tstamp(self) -> None: def test_flush_timer(self, mok_flush: Any) -> None: mok_flush.side_effect = mocked_flush - e = Emitter('0.0.0.0', method="post", buffer_size=10) + e = Emitter('0.0.0.0', buffer_size=10) ev_list = [{"a": "aa"}, {"b": "bb"}, {"c": "cc"}] for i in ev_list: e.input(i) @@ -261,7 +261,7 @@ def test_send_events_post_success(self, mok_http_post: Any) -> None: mok_success = mock.Mock(return_value="success mocked") mok_failure = mock.Mock(return_value="failure mocked") - e = Emitter('0.0.0.0', method="post", buffer_size=10, on_success=mok_success, on_failure=mok_failure) + e = Emitter('0.0.0.0', buffer_size=10, on_success=mok_success, on_failure=mok_failure) evBuffer = [{"a": "aa"}, {"b": "bb"}, {"c": "cc"}] e.send_events(evBuffer) @@ -274,7 +274,7 @@ def test_send_events_post_failure(self, mok_http_post: Any) -> None: mok_success = mock.Mock(return_value="success mocked") mok_failure = mock.Mock(return_value="failure mocked") - e = Emitter('0.0.0.0', method="post", buffer_size=10, on_success=mok_success, on_failure=mok_failure) + e = Emitter('0.0.0.0', buffer_size=10, on_success=mok_success, on_failure=mok_failure) evBuffer = [{"a": "aa"}, {"b": "bb"}, {"c": "cc"}] e.send_events(evBuffer) @@ -292,7 +292,7 @@ def test_http_post_connect_timeout_error(self, mok_post_request: Any) -> None: @mock.patch('snowplow_tracker.emitters.requests.post') def test_http_get_connect_timeout_error(self, mok_post_request: Any) -> None: mok_post_request.side_effect = ConnectTimeout - e = Emitter('0.0.0.0') + e = Emitter('0.0.0.0', method='get') get_succeeded = e.http_get({"a": "b"}) self.assertFalse(get_succeeded) @@ -366,7 +366,7 @@ def test_async_send_events_post_success(self, mok_http_post: Any) -> None: mok_success = mock.Mock(return_value="success mocked") mok_failure = mock.Mock(return_value="failure mocked") - ae = Emitter('0.0.0.0', method="post", buffer_size=10, on_success=mok_success, on_failure=mok_failure) + ae = Emitter('0.0.0.0', buffer_size=10, on_success=mok_success, on_failure=mok_failure) evBuffer = [{"a": "aa"}, {"b": "bb"}, {"c": "cc"}] ae.send_events(evBuffer) @@ -379,7 +379,7 @@ def test_async_send_events_post_failure(self, mok_http_post: Any) -> None: mok_success = mock.Mock(return_value="success mocked") mok_failure = mock.Mock(return_value="failure mocked") - ae = Emitter('0.0.0.0', method="post", buffer_size=10, on_success=mok_success, on_failure=mok_failure) + ae = Emitter('0.0.0.0', buffer_size=10, on_success=mok_success, on_failure=mok_failure) evBuffer = [{"a": "aa"}, {"b": "bb"}, {"c": "cc"}] ae.send_events(evBuffer) @@ -403,7 +403,7 @@ def test_input_unicode_post(self, mok_flush: Any) -> None: mok_flush.side_effect = mocked_flush payload = {"unicode": u'\u0107', "alsoAscii": "abc"} - ae = AsyncEmitter('0.0.0.0', method="post", buffer_size=2) + ae = AsyncEmitter('0.0.0.0', buffer_size=2) ae.input(payload) self.assertEqual(len(ae.buffer), 1) diff --git a/snowplow_tracker/tracker.py b/snowplow_tracker/tracker.py index f693e41a..16b89d8d 100644 --- a/snowplow_tracker/tracker.py +++ b/snowplow_tracker/tracker.py @@ -263,7 +263,7 @@ def track_link_click( :param element_id: ID attribute of the HTML element :type element_id: string_or_none :param element_classes: Classes of the HTML element - :type element_classes: list(str) | tuple(str,\*) | None + :type element_classes: list(str) | tuple(str,\\*) | None :param element_target: ID attribute of the HTML element :type element_target: string_or_none :param element_content: The content of the HTML element @@ -416,7 +416,7 @@ def track_form_change( :param type_: Type of data the element represents :type type_: non_empty_string, form_type :param element_classes: Classes of the HTML element - :type element_classes: list(str) | tuple(str,\*) | None + :type element_classes: list(str) | tuple(str,\\*) | None :param context: Custom context for the event :type context: context_array | None :param tstamp: Optional event timestamp in milliseconds @@ -456,7 +456,7 @@ def track_form_submit( :param form_id: ID attribute of the HTML form :type form_id: non_empty_string :param form_classes: Classes of the HTML form - :type form_classes: list(str) | tuple(str,\*) | None + :type form_classes: list(str) | tuple(str,\\*) | None :param elements: Classes of the HTML form :type elements: list(form_element) | None :param context: Custom context for the event @@ -611,7 +611,7 @@ def track_ecommerce_transaction( :param currency: The currency the price is expressed in :type currency: string_or_none :param items: The items in the transaction - :type items: list(dict(str:\*)) | None + :type items: list(dict(str:\\*)) | None :param context: Custom context for the event :type context: context_array | None :param tstamp: Optional event timestamp in milliseconds From 723f21f51eaa8e1b8c75fe8254a04709425a0812 Mon Sep 17 00:00:00 2001 From: Jack Keene Date: Thu, 3 Nov 2022 15:53:37 +0000 Subject: [PATCH 6/6] Prepare for release 0.12.0 --- CHANGES.txt | 8 ++++++++ docs/source/conf.py | 2 +- setup.py | 2 +- snowplow_tracker/_version.py | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 7139b7c5..06ba1ae9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,11 @@ +Version 0.12.0 (2022-11-03) +--------------------------- +Adds Domain Session ID and Domain Session Index to Subject class (#282) (Thanks to @cpnat) +Add support for Python 3.11 (#286) +Change default protocol to HTTPS in the Emitter (#14) +Change default method to POST in the Emitter (#289) +Update Docker base image (#283) (Thanks to @cpnat) + Version 0.11.0 (2022-10-06) --------------------------- Update README file (#264) diff --git a/docs/source/conf.py b/docs/source/conf.py index c1bab387..d906b8be 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'Alex Dean, Paul Boocock, Matus Tomlein, Jack Keene' # The full version, including alpha/beta/rc tags -release = '0.11' +release = '0.12' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 6f359ceb..cbb9e7cc 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup( name="snowplow-tracker", - version="0.11.0", + version="0.12.0", author=authors_str, author_email=authors_email_str, packages=[ diff --git a/snowplow_tracker/_version.py b/snowplow_tracker/_version.py index 43043c72..d77b9f6d 100644 --- a/snowplow_tracker/_version.py +++ b/snowplow_tracker/_version.py @@ -19,6 +19,6 @@ # License: Apache License Version 2.0 # """ -__version_info__ = (0, 11, 0) +__version_info__ = (0, 12, 0) __version__ = ".".join(str(x) for x in __version_info__) __build_version__ = __version__ + ""