From b6ad62cb6bcfe25602b766dbc597b23343a163e2 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 15:32:26 -0400
Subject: [PATCH 01/30] update gitignore for pyenv files

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index df10b506..e9bace94 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,10 @@ coverage.xml
 *.pyc
 **/__pycache__
 
+## pyenv files
+.python-version
+python/.python-version
+
 # ignore virtual environments
 python/venv
 python/.venv

From 56c18a2d0d0d6ba986bb8faa7595c4c8f34e77e3 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 16:31:46 -0400
Subject: [PATCH 02/30] ignore tox files

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index e9bace94..01b7d89d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,6 +26,10 @@ coverage.xml
 .python-version
 python/.python-version
 
+## tox files
+.tox
+python/.tox
+
 # ignore virtual environments
 python/venv
 python/.venv

From e0f42e637b8d1472fd975e46182847ba9897c7c6 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 16:42:38 -0400
Subject: [PATCH 03/30] hardcode version instead of pulling from gh

importing semver in setup caused an error bc it was being imported before requirements file could install it
---
 python/setup.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index a2d6d6a4..4301d103 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,8 +1,5 @@
 from setuptools import find_packages, setup
-from version import get_latest_git_tag
 
-# fetch the most recent version tag to use as build version
-build_version = get_latest_git_tag()
 
 # use the contents of the README file as the 'long description' for the package
 with open("./README.md", "r") as fh:
@@ -13,7 +10,7 @@
 #
 setup(
     name="dbl-tempo",
-    version=build_version,
+    version="0.1.23",
     author="Ricardo Portilla, Tristan Nixon, Max Thone, Sonali Guleria",
     author_email="labs@databricks.com",
     description="Spark Time Series Utility Package",

From c254c24a0674d5fb5e1833384cbdf96877eb8fe0 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 17:11:17 -0400
Subject: [PATCH 04/30] move coverage requirement to tox config

---
 python/requirements.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/requirements.txt b/python/requirements.txt
index fef873a3..6ddedb79 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -5,7 +5,6 @@ numpy==1.24.2
 chispa==0.9.2
 pandas==1.5.2
 pyarrow==10.0.1
-pyspark==3.2.1
 python-dateutil==2.8.2
 pytz==2022.7.1
 scipy==1.9.3
@@ -20,4 +19,3 @@ sphinx-design==0.2.0
 sphinx-panels==0.6.0
 jsonref==1.0.1
 python-dateutil==2.8.2
-coverage==6.5.0

From c41d14049a13a805a30e32c7ca174947c7d53da2 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 17:22:44 -0400
Subject: [PATCH 05/30] remove version py file since it's hardcoded in setup

---
 python/version.py | 23 -----------------------
 1 file changed, 23 deletions(-)
 delete mode 100644 python/version.py

diff --git a/python/version.py b/python/version.py
deleted file mode 100644
index d2ef5cf7..00000000
--- a/python/version.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import subprocess
-import re
-import semver
-
-
-# run a shell command and return stdout
-def run_cmd(cmd):
-    cmd_proc = subprocess.run(cmd, shell=True, capture_output=True)
-    if cmd_proc.returncode != 0:
-        raise OSError(
-            f"Shell command '{cmd}' failed with return code {cmd_proc.returncode}\n"
-            f"STDERR: {cmd_proc.stderr.decode('utf-8')}"
-        )
-    return cmd_proc.stdout.decode("utf-8").strip()
-
-
-# fetch the most recent version tag to use as build version
-def get_latest_git_tag():
-    latest_tag = run_cmd("git describe --abbrev=0 --tags")
-    build_version = re.sub("v\.?\s*", "", latest_tag)
-    # validate that this is a valid semantic version - will throw exception if not
-    semver.VersionInfo.parse(build_version)
-    return build_version

From 15b7d1feff94c42bf1e289774d3215ee18fcb722 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Wed, 19 Apr 2023 17:22:55 -0400
Subject: [PATCH 06/30] initial working tox commit

---
 python/tox.ini | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 python/tox.ini

diff --git a/python/tox.ini b/python/tox.ini
new file mode 100644
index 00000000..e39df438
--- /dev/null
+++ b/python/tox.ini
@@ -0,0 +1,93 @@
+[tox]
+requires =
+    tox>4,<5
+    virtualenv>20,<21
+    wheel>=0.38,<1
+isolated_build = true
+envlist =
+    format
+    lint
+    type
+    build-dist
+    ; Mirror Supported LTS DBR versions here: https://docs.databricks.com/release-notes/runtime/
+    ; Use correct PySpark version based on Python version present in env name
+    py37-pyspark300,
+    py38-pyspark{312,321},
+    py39-pyspark{330,332}
+skip_missing_interpreters = true
+
+[testenv]
+description = run the tests under {envname}
+package = wheel
+wheel_build_env = .pkg
+setenv =
+    COVERAGE_FILE = .coverage.{envname}
+deps =
+    pyspark300: pyspark==3.0.0
+    pyspark312: pyspark==3.1.2
+    pyspark321: pyspark==3.2.1
+    pyspark330: pyspark==3.3.0
+    pyspark332: pyspark==3.3.2
+    coverage==6.5.0
+    -rrequirements.txt
+commands =
+    python -m unittest discover -s tests -p '*_tests.py'
+
+[testenv:format]
+description = run formatters
+skipsdist = true
+skip_install = true
+deps =
+    black
+commands =
+    black {toxinidir}
+
+[testenv:lint]
+description = run linters
+skipsdist = true
+skip_install = true
+deps =
+    flake8
+    black
+commands =
+    flake8
+    black --check {toxinidir}
+
+[testenv:type]
+description = run type checks
+; todo - configure mypy
+skipsdist = true
+skip_install = true
+deps =
+    mypy
+commands =
+    mypy {toxinidir}/tempo
+
+[testenv:build-dist]
+description = build distribution
+skip_install = true
+deps =
+    build
+commands =
+    python -m build --sdist --wheel {posargs: {toxinidir}}
+
+[flake8]
+exclude =
+    .git
+    __pycache__
+    env
+    .tox
+    build
+    .venv
+    venv
+    .coverage.py
+    .coverage
+    .coveragerc
+    .eggs
+    .mypy_cache
+    .pytest_cache
+    dbl_tempo.egg-info
+max-line-length = 88
+extend-ignore =
+    ; See https://github.com/PyCQA/pycodestyle/issues/373
+    E203
\ No newline at end of file

From a3b0101ad7084382e43cc09dcf1d557133466a20 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 09:28:00 -0400
Subject: [PATCH 07/30] update required coverage versions to be more flexible

---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index e39df438..ae97ac91 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -28,7 +28,7 @@ deps =
     pyspark321: pyspark==3.2.1
     pyspark330: pyspark==3.3.0
     pyspark332: pyspark==3.3.2
-    coverage==6.5.0
+    coverage>=7,<8
     -rrequirements.txt
 commands =
     python -m unittest discover -s tests -p '*_tests.py'

From 4e899f5306005748e3903b74ff02b84c56e1f835 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 09:34:12 -0400
Subject: [PATCH 08/30] switch order formatters run

---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index ae97ac91..a7d52753 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -50,8 +50,8 @@ deps =
     flake8
     black
 commands =
-    flake8
     black --check {toxinidir}
+    flake8
 
 [testenv:type]
 description = run type checks

From dee6106d9b8d1ea853a87714f8d94e0b5bc9d174 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 09:35:04 -0400
Subject: [PATCH 09/30] ignore env specific coverage files

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 01b7d89d..baf88ec4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
 
 # coverage files
 .coverage
+.coverage.*
 coverage.xml
 
 # local delta tables

From 7e977eda4ff525e57d8c08c845d6dce4eab0cec3 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 09:45:29 -0400
Subject: [PATCH 10/30] update tox config to use coverage run instead of base
 python to execute tests

---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index a7d52753..f9fd43fd 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -31,7 +31,7 @@ deps =
     coverage>=7,<8
     -rrequirements.txt
 commands =
-    python -m unittest discover -s tests -p '*_tests.py'
+    coverage run -m unittest discover -s tests -p '*_tests.py'
 
 [testenv:format]
 description = run formatters

From 5955700d7c34107971109f2fb9c5bad5b50d96ee Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 10:07:32 -0400
Subject: [PATCH 11/30] add coverage report env to tox config

---
 python/tox.ini | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/python/tox.ini b/python/tox.ini
index f9fd43fd..e70ede48 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -71,6 +71,19 @@ deps =
 commands =
     python -m build --sdist --wheel {posargs: {toxinidir}}
 
+[testenv:coverage-report]
+deps = coverage
+skipdist = true
+skip_install = true
+commands =
+    coverage combine
+    coverage report -m
+    coverage xml
+
+[coverage:run]
+source = tempo
+parallel = true
+
 [flake8]
 exclude =
     .git

From 66f488ea1ed746d51b1c5852c276dc96b3e26794 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 10:08:58 -0400
Subject: [PATCH 12/30] update coverage-report description

---
 python/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/tox.ini b/python/tox.ini
index e70ede48..63a86d6a 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -72,6 +72,7 @@ commands =
     python -m build --sdist --wheel {posargs: {toxinidir}}
 
 [testenv:coverage-report]
+description = combine coverage data and generate reports
 deps = coverage
 skipdist = true
 skip_install = true

From 10d43942931cf239ed5c893d53c142028cf94ea5 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 10:38:49 -0400
Subject: [PATCH 13/30] update a env name for type checking

---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index 63a86d6a..6ad456b2 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -53,7 +53,7 @@ commands =
     black --check {toxinidir}
     flake8
 
-[testenv:type]
+[testenv:type-check]
 description = run type checks
 ; todo - configure mypy
 skipsdist = true

From c7b2e0cd8b3d7db50b8b3d0e81e280f3d0a6dced Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 12:44:00 -0400
Subject: [PATCH 14/30] remove black and flake8 from requirments package and
 into tox envs

---
 python/requirements.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/requirements.txt b/python/requirements.txt
index 6ddedb79..bafafbb2 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,5 +1,3 @@
-black==23.3.0
-flake8==6.0.0
 ipython==8.10.0
 numpy==1.24.2
 chispa==0.9.2

From 59bc24535df56415909617d473ff1445cfcab948 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 12:44:45 -0400
Subject: [PATCH 15/30] update contributing md file with tox instructions

---
 CONTRIBUTING.md | 91 ++++++++++++++++++++-----------------------------
 1 file changed, 37 insertions(+), 54 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3ca1e314..eee78407 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,54 +1,37 @@
-Thank you for your interest in contributing to the tempo project (the “Project”). In order to clarify the intellectual property license granted with Contributions from any person or entity who contributes to the Project, Databricks, Inc. ("Databricks") must have a Contributor License Agreement (CLA) on file that has been signed by each such Contributor (or if an entity, an authorized representative of such entity). This license is for your protection as a Contributor as well as the protection of Databricks and its users; it does not change your rights to use your own Contributions for any other purpose.
-You may sign this CLA either on your own behalf (with respect to any Contributions that are owned by you) and/or on behalf of an entity (the "Corporation") (with respect to any Contributions that are owned by such Corporation (e.g., those Contributions you make during the performance of your employment duties to the Corporation)).  Please mark the corresponding box below.
-You accept and agree to the following terms and conditions for Your present and future Contributions submitted to Databricks. Except for the licenses granted herein to Databricks, You reserve all right, title, and interest in and to Your Contributions.
-Definitions.
-"You" (or "Your") shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Databricks. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
-"Contribution" shall mean the code, documentation or any original work of authorship, including any modifications or additions to an existing work, that is submitted by You to Databricks for inclusion in, or documentation of, any of the products owned or managed by Databricks, including the Project, whether on, before or after the date You sign this CLA. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to Databricks or its representatives, including but not limited to communication on electronic mailing lists, source code control systems (e.g., Github), and issue tracking systems that are managed by, or on behalf of, Databricks for the purpose of discussing and improving the Project, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
-Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Databricks a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense (through multiple tiers), and distribute Your Contributions and such derivative works.  For the avoidance of doubt, and without limitation, this includes, at our option, the right to sublicense this license to recipients or users of any products or services (including software) distributed or otherwise made available (e.g., by SaaS offering) by Databricks (each, a “Downstream Recipient”).
-Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Databricks a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable, sublicensable (through multiple tiers) (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Contribution in whole or in part, alone or in combination with any other products or services (including for the avoidance of doubt the Project), where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by combination of Your Contribution(s) with the Project to which such Contribution(s) was submitted.  For the avoidance of doubt, and without limitation, this includes, at our option, the right to sublicense this license to Downstream Recipients.
-Authorized Users. If you are signing this CLA on behalf of a Corporation, you may also add additional designated employees of the Corporation who will be covered by this CLA without the need to separately sign it (“Authorized Users”).  Your Primary Point of Contact (you or the individual specified below) may add additional Authorized Users at any time by contacting Databricks at cla@databricks.com (or such other method as Databricks informs you).
-Representations. You represent that:
-You are legally entitled to grant the above licenses, and, if You are signing on behalf of a Corporation and have added any Authorized Users, You represent further that each employee of the Corporation designated by You is authorized to submit Contributions on behalf of the Corporation;
-each of Your Contributions is Your original creation;
-to your knowledge, Your Contributions do not infringe or otherwise misappropriate the intellectual property rights of a third person; and
-you will not assert any moral rights in your Contribution against us or any Downstream Recipients.
-Support. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, and except as specified above, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
-Notification. It is your responsibility to notify Databricks when any change is required to the list of Authorized Users, or to the Corporation's Primary Point of Contact with Databricks.  You agree to notify Databricks of any facts or circumstances of which you become aware that would make the representations or warranties herein inaccurate in any respect.
-This CLA is governed by the laws of the State of California and applicable U.S. Federal law. Any choice of law rules will not apply.
-Please check one of the applicable statement below. Please do NOT mark both statements:
- 	I am signing on behalf of myself as an individual and no other person or entity, including my employer, has or will have rights with respect my Contributions.
- 	I am signing on behalf of my employer or a legal entity and I have the actual authority to contractually bind such entity (the Corporation).
-
-Name*:
-
-
-Corporation Entity Name (if applicable):
-
-
-Title or Role (if applicable):
-
-
-Mailing Address*:
-
-
-Email*:
-
-
-Signature*:
-
-
-Date*:
-
-
-Github Username (if applicable):
-
-
-Primary Point of Contact (if not you) (please provide name and email and Github username, if applicable):
-
-
-Authorized Users (please list Github usernames):**
-
-
-
-* Required field
-** Please note that Authorized Users may not be immediately be granted authorization to submit Contributions; should more than one individual attempt to sign a CLA on behalf of a Corporation, the first such CLA will apply and later CLAs will be deemed void.
\ No newline at end of file
+# Tox Setup instructions
+A brief description of each configured environment can be found by running `tox list` or in the `tox.ini` file.
+
+## Create a development environment
+Run the following command in your terminal to create a virtual environment in the `.venv` folder:
+```bash
+tox --devenv .venv -e {environment-name}
+```
+The `—devenv` flag tells `tox` to create a development environment, and `.venv` is the folder where the virtual environment will be created.
+Pre-defined environments can be found within the `tox.ini` file for different Python versions and their corresponding PySpark version. They include:
+- py37-pyspark300
+- py38-pyspark312
+- py38-pyspark321
+- py39-pyspark330
+- py39-pyspark332
+
+## Run tests locally for one or more environments
+You can run tests locally for one or more environments defined enviornments without setting up a development environment first.
+
+### To run tests for a single environment, use the `-e` flag followed by the environment name:
+```bash
+tox -e {environment-name}
+```
+
+### To run tests for multiple environments, specify the environment names separated by commas:
+```bash
+tox -e {environment-name1, environment-name2, etc.}
+```
+This will run tests for all listed environments.
+
+### Run additional checks locally
+`tox` has special environments for additional checks that must be performed as part of the PR process. These include formatting, linting, type checking, etc.
+These environments are also defined in the `tox.ini`file and skip installing dependencies listed in the `requirements.txt` file and building the distribution when those are not required . They can be specified using the `-e` flag:
+	●	format
+	●	lint
+	●	type-check
+	●	coverage-report

From bbc0406e46774fe20e0ebcbab4d5867e971df382 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 12:51:42 -0400
Subject: [PATCH 16/30] add pyenv and tox instructions to CONTRIBUTING md

---
 CONTRIBUTING.md | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index eee78407..c157c941 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,5 +1,25 @@
 # Tox Setup instructions
-A brief description of each configured environment can be found by running `tox list` or in the `tox.ini` file.
+
+`tox` is a testing tool that helps you automate and standardize testing in Python across multiple environments.
+
+`pyenv`that allows you to manage multiple versions of Python on your computer and easily switch between them.
+
+Since `tox` supports creating virtual environments using multiple Python versions, it is recommended to use `pyenv` to manage Python versions on your computer.
+
+Install both tox and pyenv packages:
+```bash
+pip install -U tox pyenv
+pyenv install 3.7 3.8 3.9
+```
+
+Within `python` folder, run the below command to create a `.python-version` file that will tell `pyenv` which Python version to use when running commands in this directory:
+```bash
+pyenv local 3.7 3.8 3.9
+```
+
+This allows `tox` to create virtual environments using any of the Python versions listed in the `.python-version` file.
+
+A brief description of each managed `tox` environment can be found by running `tox list` or in the `tox.ini` file.
 
 ## Create a development environment
 Run the following command in your terminal to create a virtual environment in the `.venv` folder:

From 1ab4bd084380fbdf94048c42fd4b1886e68ba2a5 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <22798188+R7L208@users.noreply.github.com>
Date: Mon, 24 Apr 2023 12:59:28 -0400
Subject: [PATCH 17/30] Update test.yml

---
 .github/workflows/test.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a0403f73..77b25bef 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
   black-linting:
     runs-on: ubuntu-latest
     name: Black Lint
-    steps:
+   steps:
       - uses: actions/checkout@v2
       - uses: psf/black@stable
         with:
@@ -54,9 +54,8 @@ jobs:
     - name: Generate coverage report
       working-directory: ./python
       run: |
-        pip install -r requirements.txt
-        pip install coverage
+        python -I -m pip install 'coverage<8,>=7' pyspark==3.2.1 -r requirements.txt
         coverage run -m unittest discover -s tests -p '*_tests.py'
         coverage xml
     - name: Publish test coverage
-      uses: codecov/codecov-action@v1
\ No newline at end of file
+      uses: codecov/codecov-action@v1

From 92d483624649cc64fe682b6b15416e2a93aac004 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 13:27:45 -0400
Subject: [PATCH 18/30] adding comment to force testing on GH actions

---
 python/tempo/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/tempo/__init__.py b/python/tempo/__init__.py
index da4a6c12..e1851d7a 100644
--- a/python/tempo/__init__.py
+++ b/python/tempo/__init__.py
@@ -1,2 +1,4 @@
 from tempo.tsdf import TSDF  # noqa: F401
 from tempo.utils import display  # noqa: F401
+
+#adding comment to force testing on GH actions
\ No newline at end of file

From 0d658e9df820e9b536a7bcdcbac96bcbf9680458 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Mon, 24 Apr 2023 13:30:14 -0400
Subject: [PATCH 19/30] removing comment to force testing on GH actions

---
 python/tempo/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/python/tempo/__init__.py b/python/tempo/__init__.py
index e1851d7a..da4a6c12 100644
--- a/python/tempo/__init__.py
+++ b/python/tempo/__init__.py
@@ -1,4 +1,2 @@
 from tempo.tsdf import TSDF  # noqa: F401
 from tempo.utils import display  # noqa: F401
-
-#adding comment to force testing on GH actions
\ No newline at end of file

From fae16711bb4c6ef87bd63eb9d498fc6eb207f225 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <22798188+R7L208@users.noreply.github.com>
Date: Mon, 24 Apr 2023 13:34:56 -0400
Subject: [PATCH 20/30] Update test.yml to fix error caused by typo

---
 .github/workflows/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 77b25bef..833264b7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
   black-linting:
     runs-on: ubuntu-latest
     name: Black Lint
-   steps:
+    steps:
       - uses: actions/checkout@v2
       - uses: psf/black@stable
         with:

From 571746ba35bc44dec0aee6bcd43c48cad8cd0e23 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Tue, 25 Apr 2023 09:48:18 -0400
Subject: [PATCH 21/30] Revert "remove version py file since it's hardcoded in
 setup"

This reverts commit c41d14049a13a805a30e32c7ca174947c7d53da2.
---
 python/version.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 python/version.py

diff --git a/python/version.py b/python/version.py
new file mode 100644
index 00000000..d2ef5cf7
--- /dev/null
+++ b/python/version.py
@@ -0,0 +1,23 @@
+import subprocess
+import re
+import semver
+
+
+# run a shell command and return stdout
+def run_cmd(cmd):
+    cmd_proc = subprocess.run(cmd, shell=True, capture_output=True)
+    if cmd_proc.returncode != 0:
+        raise OSError(
+            f"Shell command '{cmd}' failed with return code {cmd_proc.returncode}\n"
+            f"STDERR: {cmd_proc.stderr.decode('utf-8')}"
+        )
+    return cmd_proc.stdout.decode("utf-8").strip()
+
+
+# fetch the most recent version tag to use as build version
+def get_latest_git_tag():
+    latest_tag = run_cmd("git describe --abbrev=0 --tags")
+    build_version = re.sub("v\.?\s*", "", latest_tag)
+    # validate that this is a valid semantic version - will throw exception if not
+    semver.VersionInfo.parse(build_version)
+    return build_version

From 2a0b2fbb68f4d6f84f5850970cd5715b7fd4c95a Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Tue, 25 Apr 2023 09:48:52 -0400
Subject: [PATCH 22/30] Revert "hardcode version instead of pulling from gh"

This reverts commit e0f42e637b8d1472fd975e46182847ba9897c7c6.
---
 python/setup.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python/setup.py b/python/setup.py
index 4301d103..a2d6d6a4 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,5 +1,8 @@
 from setuptools import find_packages, setup
+from version import get_latest_git_tag
 
+# fetch the most recent version tag to use as build version
+build_version = get_latest_git_tag()
 
 # use the contents of the README file as the 'long description' for the package
 with open("./README.md", "r") as fh:
@@ -10,7 +13,7 @@
 #
 setup(
     name="dbl-tempo",
-    version="0.1.23",
+    version=build_version,
     author="Ricardo Portilla, Tristan Nixon, Max Thone, Sonali Guleria",
     author_email="labs@databricks.com",
     description="Spark Time Series Utility Package",

From 6004da33c3db648105c3f52b1599fb271ca214b3 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Tue, 25 Apr 2023 11:07:25 -0400
Subject: [PATCH 23/30] add semver as a dep for testenv in tox ini

---
 python/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/tox.ini b/python/tox.ini
index 6ad456b2..315b0f3d 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -16,6 +16,7 @@ envlist =
     py39-pyspark{330,332}
 skip_missing_interpreters = true
 
+
 [testenv]
 description = run the tests under {envname}
 package = wheel

From 6093075d3030d2be6b7e6758171fc329270d9538 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Tue, 25 Apr 2023 11:23:53 -0400
Subject: [PATCH 24/30] add pyproject toml to install semver as build
 requirement

---
 python/pyproject.toml | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 python/pyproject.toml

diff --git a/python/pyproject.toml b/python/pyproject.toml
new file mode 100644
index 00000000..d7a6d464
--- /dev/null
+++ b/python/pyproject.toml
@@ -0,0 +1,2 @@
+[build-system]
+requires = ["semver"]  # PEP 518 - what is required to build

From b4a170a1db2e03524bda5deccee9759c178075a7 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Thu, 27 Apr 2023 11:43:48 -0400
Subject: [PATCH 25/30] update type default env to type-check

this matches the actual env name since type was renamed to type-check
---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index 315b0f3d..4aa7b916 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -7,7 +7,7 @@ isolated_build = true
 envlist =
     format
     lint
-    type
+    type-check
     build-dist
     ; Mirror Supported LTS DBR versions here: https://docs.databricks.com/release-notes/runtime/
     ; Use correct PySpark version based on Python version present in env name

From 1d30f56482a053795ba3887a22713ae46f119afc Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Thu, 27 Apr 2023 12:16:55 -0400
Subject: [PATCH 26/30] move all flake8 configs to tox.ini file

flake8 was returning different results locally in virtual env and using tox. This is because the configuration in tox.ini takes precedence over the .flake8 config file. In addition, flake8 no longer supports a global config file after version 4
---
 python/.flake8 | 18 ------------------
 python/tox.ini | 18 +++++++++++++-----
 2 files changed, 13 insertions(+), 23 deletions(-)
 delete mode 100644 python/.flake8

diff --git a/python/.flake8 b/python/.flake8
deleted file mode 100644
index 6df84325..00000000
--- a/python/.flake8
+++ /dev/null
@@ -1,18 +0,0 @@
-[flake8]
-max-line-length = 88
-builtins = dbutils
-extend-ignore = 
-    # Line too long (88 characters) (E501)
-    E501,
-    # Name may be undefined, or defined from star imports: module (F405)
-    F405,
-    # Invalid escape sequence 'x' (W605)
-    W605,
-    # 'from module import *' used; unable to detect undefined names (F403)
-    F403
-exclude =
-    __pycache__
-    env
-    .env
-    venv
-    .venv
\ No newline at end of file
diff --git a/python/tox.ini b/python/tox.ini
index 4aa7b916..463fe5db 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -52,7 +52,7 @@ deps =
     black
 commands =
     black --check {toxinidir}
-    flake8
+    flake8 {toxinidir}
 
 [testenv:type-check]
 description = run type checks
@@ -87,6 +87,17 @@ source = tempo
 parallel = true
 
 [flake8]
+max-line-length = 88
+builtins = dbutils
+extend-ignore =
+    # Line too long (88 characters) (E501)
+    E501,
+    # Name may be undefined, or defined from star imports: module (F405)
+    F405,
+    # Invalid escape sequence 'x' (W605)
+    W605,
+    # 'from module import *' used; unable to detect undefined names (F403)
+    F403
 exclude =
     .git
     __pycache__
@@ -102,7 +113,4 @@ exclude =
     .mypy_cache
     .pytest_cache
     dbl_tempo.egg-info
-max-line-length = 88
-extend-ignore =
-    ; See https://github.com/PyCQA/pycodestyle/issues/373
-    E203
\ No newline at end of file
+    .tox

From 681c97c2b3a1bdfcebd9bf4187d7fec6a398e033 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Thu, 27 Apr 2023 12:19:37 -0400
Subject: [PATCH 27/30] remove toxinidir from flake8 command

---
 python/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/tox.ini b/python/tox.ini
index 463fe5db..3262dd1d 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -52,7 +52,7 @@ deps =
     black
 commands =
     black --check {toxinidir}
-    flake8 {toxinidir}
+    flake8
 
 [testenv:type-check]
 description = run type checks

From 108c3f112b023bfe8602f47291237a6ae7138559 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Thu, 27 Apr 2023 12:21:08 -0400
Subject: [PATCH 28/30] remove all flake8 config from tox ini file and back
 into flake8 config so that GH actions aren't broken

---
 python/.flake8 | 28 ++++++++++++++++++++++++++++
 python/tox.ini | 29 -----------------------------
 2 files changed, 28 insertions(+), 29 deletions(-)
 create mode 100644 python/.flake8

diff --git a/python/.flake8 b/python/.flake8
new file mode 100644
index 00000000..8974cbf4
--- /dev/null
+++ b/python/.flake8
@@ -0,0 +1,28 @@
+[flake8]
+max-line-length = 88
+builtins = dbutils
+extend-ignore =
+    # Line too long (88 characters) (E501)
+    E501,
+    # Name may be undefined, or defined from star imports: module (F405)
+    F405,
+    # Invalid escape sequence 'x' (W605)
+    W605,
+    # 'from module import *' used; unable to detect undefined names (F403)
+    F403
+exclude =
+    .git
+    __pycache__
+    env
+    .tox
+    build
+    .venv
+    venv
+    .coverage.py
+    .coverage
+    .coveragerc
+    .eggs
+    .mypy_cache
+    .pytest_cache
+    dbl_tempo.egg-info
+    .tox
\ No newline at end of file
diff --git a/python/tox.ini b/python/tox.ini
index 3262dd1d..6b8da542 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -85,32 +85,3 @@ commands =
 [coverage:run]
 source = tempo
 parallel = true
-
-[flake8]
-max-line-length = 88
-builtins = dbutils
-extend-ignore =
-    # Line too long (88 characters) (E501)
-    E501,
-    # Name may be undefined, or defined from star imports: module (F405)
-    F405,
-    # Invalid escape sequence 'x' (W605)
-    W605,
-    # 'from module import *' used; unable to detect undefined names (F403)
-    F403
-exclude =
-    .git
-    __pycache__
-    env
-    .tox
-    build
-    .venv
-    venv
-    .coverage.py
-    .coverage
-    .coveragerc
-    .eggs
-    .mypy_cache
-    .pytest_cache
-    dbl_tempo.egg-info
-    .tox

From 5e714a91446ffafd793639df4367ee6f5a6395c2 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Fri, 28 Apr 2023 13:34:24 -0400
Subject: [PATCH 29/30] remove ignore for F403 for flake8 config and violations

---
 python/.flake8                  |  2 --
 python/tests/interpol_tests.py  |  3 +--
 python/tests/intervals_tests.py |  4 +++-
 python/tests/utils_tests.py     | 15 ++++++++++++++-
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/python/.flake8 b/python/.flake8
index 8974cbf4..d0ec008c 100644
--- a/python/.flake8
+++ b/python/.flake8
@@ -8,8 +8,6 @@ extend-ignore =
     F405,
     # Invalid escape sequence 'x' (W605)
     W605,
-    # 'from module import *' used; unable to detect undefined names (F403)
-    F403
 exclude =
     .git
     __pycache__
diff --git a/python/tests/interpol_tests.py b/python/tests/interpol_tests.py
index ec7d0a29..b8b20f87 100644
--- a/python/tests/interpol_tests.py
+++ b/python/tests/interpol_tests.py
@@ -1,10 +1,9 @@
 import unittest
 
-from pyspark.sql.types import *
+from pyspark.sql.dataframe import DataFrame
 
 from tempo.interpol import Interpolation
 from tempo.tsdf import TSDF
-from tempo.utils import *
 
 from tests.tsdf_tests import SparkTest
 
diff --git a/python/tests/intervals_tests.py b/python/tests/intervals_tests.py
index 841b4dd2..9a10ba26 100644
--- a/python/tests/intervals_tests.py
+++ b/python/tests/intervals_tests.py
@@ -1,4 +1,6 @@
-from tempo.intervals import *
+from pyspark.sql.dataframe import DataFrame
+
+from tempo.intervals import IntervalsDF
 from tests.tsdf_tests import SparkTest
 from pyspark.sql.utils import AnalysisException
 import pyspark.sql.functions as f
diff --git a/python/tests/utils_tests.py b/python/tests/utils_tests.py
index 24c16d69..f79aa0e9 100644
--- a/python/tests/utils_tests.py
+++ b/python/tests/utils_tests.py
@@ -1,8 +1,21 @@
+import os
+import warnings
 from io import StringIO
 import sys
 import unittest
 
-from tempo.utils import *
+from tempo.utils import (
+    IS_DATABRICKS,
+    display,
+    display_improvised,
+    display_html_improvised,
+    ENV_CAN_RENDER_HTML,
+    display_unavailable,
+    calculate_time_horizon,
+    display_html,
+    get_display_df,
+)
+
 from tests.tsdf_tests import SparkTest
 from unittest import mock
 

From 8279a5f36a15509b0708ba9144f1bb772d019880 Mon Sep 17 00:00:00 2001
From: Lorin Dawson <lorin@databricks.com>
Date: Fri, 28 Apr 2023 13:52:17 -0400
Subject: [PATCH 30/30] add noqa for utils import

---
 python/tests/utils_tests.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/python/tests/utils_tests.py b/python/tests/utils_tests.py
index f79aa0e9..c5cdb808 100644
--- a/python/tests/utils_tests.py
+++ b/python/tests/utils_tests.py
@@ -1,20 +1,8 @@
-import os
-import warnings
 from io import StringIO
 import sys
 import unittest
 
-from tempo.utils import (
-    IS_DATABRICKS,
-    display,
-    display_improvised,
-    display_html_improvised,
-    ENV_CAN_RENDER_HTML,
-    display_unavailable,
-    calculate_time_horizon,
-    display_html,
-    get_display_df,
-)
+from tempo.utils import *  # noqa: F403
 
 from tests.tsdf_tests import SparkTest
 from unittest import mock