diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 94fcc59cf..d1d480cc8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.1.12 +current_version = 4.1.13 commit = False tag = False diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 97dc620be..bfaa3109e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,7 +1,5 @@ # style(black): format cdc acquisition 980b0b7e80c7923b79e14fee620645e680785703 -# style(black): format covidcast_nowcast acquisition -9e6ff16f599e8feec34a08dd1bddbc5eae347b55 # style(black): format ecdc acquisition d1141d904da4e62992b97c92d5caebd8fadffd42 # style(black): format flusurv acquisition diff --git a/.github/workflows/performance-tests-one-time.yml b/.github/workflows/performance-tests-one-time.yml new file mode 100644 index 000000000..f11c02c4d --- /dev/null +++ b/.github/workflows/performance-tests-one-time.yml @@ -0,0 +1,106 @@ +name: One-time performance testing - 26th October 2023 + +# Run "At every 30th minute on day-of-month 26 in October" +on: + schedule: + - cron: '*/30 * 26 10 *' + +# Add some extra perms to comment on a PR +permissions: + pull-requests: write + contents: read + +jobs: + run-perftests: + # Run this on Delphi's self-hosted runner + runs-on: self-hosted + outputs: + request_count: ${{ steps.output.outputs.request_count }} + failure_count: ${{ steps.output.outputs.failure_count }} + med_time: ${{ steps.output.outputs.med_time }} + avg_time: ${{ steps.output.outputs.avg_time }} + min_time: ${{ steps.output.outputs.min_time }} + max_time: ${{ steps.output.outputs.max_time }} + requests_per_sec: ${{ steps.output.outputs.requests_per_sec }} + steps: + - name: Set up WireGuard + uses: egor-tensin/setup-wireguard@v1.2.0 + with: + endpoint: '${{ secrets.WG_PERF_ENDPOINT }}' + endpoint_public_key: '${{ secrets.WG_PERF_ENDPOINT_PUBLIC_KEY }}' + ips: '${{ secrets.WG_PERF_IPS }}' + allowed_ips: '${{ secrets.WG_PERF_ALLOWED_IPS }}' + private_key: '${{ secrets.WG_PERF_PRIVATE_KEY }}' + - name: Clean files from previous runs + uses: AutoModality/action-clean@v1 + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up repository # mimics install.sh in the README except that delphi is cloned from the PR rather than main + run: | + cd .. + rm -rf driver + mkdir -p driver/repos/delphi + cd driver/repos/delphi + git clone https://github.com/cmu-delphi/operations + git clone https://github.com/cmu-delphi/utils + git clone https://github.com/cmu-delphi/flu-contest + git clone https://github.com/cmu-delphi/nowcast + cd ../../ + + cd .. + cp -R delphi-epidata driver/repos/delphi/delphi-epidata + cd - + + ln -s repos/delphi/delphi-epidata/dev/local/Makefile + - name: Build & run epidata + run: | + cd ../driver + sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second" + sudo make redis + - name: Check out delphi-admin + uses: actions/checkout@v3 + with: + repository: cmu-delphi/delphi-admin + token: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }} + path: delphi-admin + - name: Build & run Locust + continue-on-error: true # sometimes ~2-5 queries fail, we shouldn't end the run if that's the case + run: | + cd delphi-admin/load-testing/locust + docker build -t locust . + export CSV=v4-requests-small.csv + touch output_stats.csv && chmod 666 output_stats.csv + touch output_stats_history.csv && chmod 666 output_stats_history.csv + touch output_failures.csv && chmod 666 output_failures.csv + touch output_exceptions.csv && chmod 666 output_exceptions.csv + docker run --net=host -v $PWD:/mnt/locust -e CSV="/mnt/locust/${CSV}" locust -f /mnt/locust/v4.py --host http://127.0.0.1:10080/ --users 10 --spawn-rate 1 --headless -i "$(cat ${CSV} | wc -l)" --csv=/mnt/locust/output + - name: Produce output for summary + id: output + uses: jannekem/run-python-script-action@v1 + with: + script: | + import os + + def write_string(name, value): + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + print(f'{name}={value}', file=fh) + + def write_float(name, value): + write_string(name, "{:.2f}".format(float(value))) + + with open("delphi-admin/load-testing/locust/output_stats.csv", "r", encoding="utf-8", errors="ignore") as scraped: + final_line = scraped.readlines()[-1].split(",") + write_string('request_count', final_line[2]) + write_string('failure_count', final_line[3]) + write_float('med_time', final_line[4]) + write_float('avg_time', final_line[5]) + write_float('min_time', final_line[6]) + write_float('max_time', final_line[7]) + write_float('requests_per_sec', final_line[9]) + + - name: Archive results as artifacts + uses: actions/upload-artifact@v3 + with: + name: locust-output + path: | + delphi-admin/load-testing/locust/output_*.csv diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 11d085337..ec30856d0 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -63,7 +63,7 @@ jobs: - name: Build & run epidata run: | cd ../driver - sudo make web sql="${{ secrets.DB_CONN_STRING }}" + sudo make web sql="${{ secrets.DB_CONN_STRING }}" rate_limit="999999/second" sudo make redis - name: Check out delphi-admin uses: actions/checkout@v3 diff --git a/deploy.json b/deploy.json index c1ee1ab07..369000026 100644 --- a/deploy.json +++ b/deploy.json @@ -208,15 +208,6 @@ "add-header-comment": true }, - "// acquisition - covidcast_nowcast", - { - "type": "move", - "src": "src/acquisition/covidcast_nowcast/", - "dst": "[[package]]/acquisition/covidcast_nowcast/", - "match": "^.*\\.(py)$", - "add-header-comment": true - }, - "// maintenance", { "type": "move", diff --git a/dev/local/Makefile b/dev/local/Makefile index fb02668ee..84c308257 100644 --- a/dev/local/Makefile +++ b/dev/local/Makefile @@ -37,6 +37,7 @@ # test= Only runs tests in the directories provided here, e.g. # repos/delphi/delphi-epidata/tests/acquisition/covidcast # sql= Overrides the default SQL connection string. +# rate_limit= Overrides the default rate limit for API requests. # Set optional argument defaults @@ -56,6 +57,14 @@ else sqlalchemy_uri:=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata endif +ifdef rate_limit +# Notation found here: https://flask-limiter.readthedocs.io/en/stable/#rate-limit-string-notation + rate_limit_settings:=--env "RATE_LIMIT=$(rate_limit)" +else +# Default behavior is to set the rate limit to "5/hour" for API key tests via this environment variable + rate_limit_settings:=--env "TESTING_MODE=True" +endif + SHELL:=/bin/sh # Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655 @@ -104,7 +113,7 @@ web: --env "REDIS_PASSWORD=1234" \ --env "API_KEY_ADMIN_PASSWORD=test_admin_password" \ --env "API_KEY_REGISTER_WEBHOOK_TOKEN=abc" \ - --env "TESTING_MODE=True" \ + $(rate_limit_settings) \ --network delphi-net --name delphi_web_epidata \ delphi_web_epidata >$(LOG_WEB) 2>&1 & diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg index 1d08efd6f..99ec9583f 100644 --- a/dev/local/setup.cfg +++ b/dev/local/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = Delphi Development -version = 4.1.12 +version = 4.1.13 [options] packages = @@ -13,7 +13,6 @@ packages = delphi.epidata.acquisition.covid_hosp.state_daily delphi.epidata.acquisition.covid_hosp.state_timeseries delphi.epidata.acquisition.covidcast - delphi.epidata.acquisition.covidcast_nowcast delphi.epidata.acquisition.ecdc delphi.epidata.acquisition.flusurv delphi.epidata.acquisition.fluview diff --git a/integrations/acquisition/covidcast_nowcast/__init__.py b/integrations/acquisition/covidcast_nowcast/__init__.py deleted file mode 100644 index e197f3ec4..000000000 --- a/integrations/acquisition/covidcast_nowcast/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -import os - -sys.path.append(os.getcwd()) diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py deleted file mode 100644 index bf1a0f9a0..000000000 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Integration tests for covidcast's CSV-to-database uploading.""" - -# standard library -from datetime import date -import os -import unittest -from unittest.mock import patch -from functools import partialmethod -from datetime import date - -# third party -import mysql.connector -import epiweeks as epi - - -# first party -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.acquisition.covidcast_nowcast.load_sensors import main -from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter -import delphi.operations.secrets as secrets - -# py3tester coverage target (equivalent to `import *`) -__test_target__ = 'delphi.epidata.acquisition.covidcast_nowcast.load_sensors' - -FIXED_ISSUE_IMPORTER = partialmethod(CsvImporter.find_csv_files, - issue=(date(2020, 4, 21), epi.Week.fromdate(date(2020, 4, 21))) - ) - - -class CsvUploadingTests(unittest.TestCase): - """Tests covidcast nowcast CSV uploading.""" - - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - - @patch('delphi.epidata.acquisition.covidcast_nowcast.load_sensors.CsvImporter.find_csv_files', - new=FIXED_ISSUE_IMPORTER) - def test_uploading(self): - """Scan, parse, upload, archive, serve, and fetch a covidcast_nowcast signal.""" - - # print full diff if something unexpected comes out - self.maxDiff=None - - receiving_dir = '/common/covidcast_nowcast/receiving/issue_20200421/src/' - success_dir = '/common/covidcast_nowcast/archive/successful/issue_20200421/src/' - failed_dir = '/common/covidcast_nowcast/archive/failed/issue_20200421/src/' - os.makedirs(receiving_dir, exist_ok=True) - - # valid - with open(receiving_dir + '20200419_state_sig.csv', 'w') as f: - f.write('sensor_name,geo_value,value\n') - f.write('testsensor,ca,1\n') - - # invalid filename - with open(receiving_dir + 'hello.csv', 'w') as f: - f.write('file name is wrong\n') - - # upload CSVs - main() - - # check files moved correctly - self.assertTrue( - os.path.isfile(success_dir + '20200419_state_sig.csv') - ) - self.assertTrue( - os.path.isfile(failed_dir + 'hello.csv') - ) - with self.assertRaises(Exception): - # no data, will cause pandas to raise error - with open(receiving_dir + '20200419_state_empty.csv', 'w') as f: - f.write('') - main() - self.assertTrue( - os.path.isfile(failed_dir + '20200419_state_empty.csv') - ) - - # check data uploaded - response = Epidata.covidcast_nowcast( - 'src', 'sig', 'testsensor', 'day', 'state', 20200419, 'ca') - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'time_value': 20200419, - 'geo_value': 'ca', - 'value': 1, - 'issue': 20200421, - 'lag': 2, - 'signal': 'sig', - }], - 'message': 'success', - }) - - @patch('delphi.epidata.acquisition.covidcast_nowcast.load_sensors.CsvImporter.find_csv_files', - new=FIXED_ISSUE_IMPORTER) - def test_duplicate_row(self): - """Test duplicate unique keys are updated.""" - - # print full diff if something unexpected comes out - self.maxDiff=None - - receiving_dir = '/common/covidcast_nowcast/receiving/issue_20200425/src/' - os.makedirs(receiving_dir, exist_ok=True) - - with open(receiving_dir + '20200419_state_sig.csv', 'w') as f: - f.write('sensor_name,geo_value,value\n') - f.write('testsensor,ca,1\n') - main() - with open(receiving_dir + '20200419_state_sig.csv', 'w') as f: - f.write('sensor_name,geo_value,value\n') - f.write('testsensor,ca,2\n') - main() - - # most most recent value is the one stored - response = Epidata.covidcast_nowcast( - 'src', 'sig', 'testsensor', 'day', 'state', 20200419, 'ca') - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'time_value': 20200419, - 'geo_value': 'ca', - 'value': 2, - 'issue': 20200425, - 'lag': 6, - 'signal': 'sig', - }], - 'message': 'success', - }) diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py deleted file mode 100644 index 84fc0e080..000000000 --- a/integrations/client/test_nowcast.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Integration tests for delphi_epidata.py.""" - -# standard library -import unittest - -# third party -import mysql.connector - -# first party -from delphi.epidata.client.delphi_epidata import Epidata -import delphi.operations.secrets as secrets - -# py3tester coverage target -__test_target__ = 'delphi.epidata.client.delphi_epidata' - -class DelphiEpidataPythonClientNowcastTests(unittest.TestCase): - """Tests the Python client.""" - - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database and clear relevant tables - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values ("key", "email")') - - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' - Epidata.auth = ('epidata', 'key') - - # use the local instance of the epidata database - secrets.db.host = 'delphi_database_epidata' - secrets.db.epi = ('user', 'pass') - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - - def test_covidcast_nowcast(self): - """Test that the covidcast_nowcast endpoint returns expected data.""" - - # insert dummy data - self.cur.execute(f'''insert into covidcast_nowcast values - (0, 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 3.5, 20200101, 2), - (0, 'src', 'sig2', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 2.5, 20200101, 2), - (0, 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 1.5, 20200102, 2)''') - self.cnx.commit() - - # fetch data - response = Epidata.covidcast_nowcast( - 'src', ['sig1', 'sig2'], 'sensor', 'day', 'county', 20200101, '01001') - - # request two signals - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 1.5, - 'issue': 20200102, - 'lag': 2, - 'signal': 'sig1', - }, { - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 2.5, - 'issue': 20200101, - 'lag': 2, - 'signal': 'sig2', - }], - 'message': 'success', - }) - - # request range of issues - response = Epidata.covidcast_nowcast( - 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', - issues=Epidata.range(20200101, 20200102)) - - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 3.5, - 'issue': 20200101, - 'lag': 2, - 'signal': 'sig1', - }, { - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 1.5, - 'issue': 20200102, - 'lag': 2, - 'signal': 'sig1', - }], - 'message': 'success', - }) - - # request as_of - response = Epidata.covidcast_nowcast( - 'src', 'sig1', 'sensor', 'day', 'county', 20200101, '01001', - as_of=20200101) - - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 3.5, - 'issue': 20200101, - 'lag': 2, - 'signal': 'sig1', - }], - 'message': 'success', - }) - - # request unavailable data - response = Epidata.covidcast_nowcast( - 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') - - self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 8cb50016a..27b992f17 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -69,7 +69,7 @@ def test_multiples_allowed_signal_two_multiples(self): def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" params = { - "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", + "signal": "hospital-admissions:smoothed_covid19", "time_type": "day", "geo_type": "state", "geo_value": "pa,ny", @@ -83,7 +83,7 @@ def test_multiples_non_allowed_signal(self): def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { - "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", + "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_covid19", "time_type": "day", "geo_type": "state", "geo_value": "pa", diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py deleted file mode 100644 index 32445afdf..000000000 --- a/integrations/server/test_covidcast_nowcast.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Integration tests for the `covidcast_nowcast` endpoint.""" - -# standard library -import unittest - -# third party -import mysql.connector -import requests - - -# use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata' -AUTH = ('epidata', 'key') - - -class CovidcastTests(unittest.TestCase): - """Tests the `covidcast` endpoint.""" - - def setUp(self): - """Perform per-test setup.""" - - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='epidata') - cur = cnx.cursor() - cur.execute('truncate table covidcast_nowcast') - cur.execute('delete from api_user') - cur.execute('insert into api_user(api_key, email) values("key", "email")') - cnx.commit() - cur.close() - - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - - def tearDown(self): - """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() - - @staticmethod - def _make_request(params: dict): - response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) - response.raise_for_status() - return response.json() - - def test_query(self): - """Query nowcasts using default and specified issue.""" - - self.cur.execute( - f'''insert into covidcast_nowcast values - (0, 'src', 'sig', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 3.5, 20200101, 2), - (0, 'src', 'sig', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 2.5, 20200102, 2), - (0, 'src', 'sig', 'sensor', 'day', 'county', 20200101, '01001', 12345678, 1.5, 20200103, 2)''') - - self.cnx.commit() - # make the request with specified issue date - params={ - 'data_source': 'src', - 'signals': 'sig', - 'sensor_names': 'sensor', - 'time_type': 'day', - 'geo_type': 'county', - 'time_values': 20200101, - 'geo_value': '01001', - 'issues': 20200101 - } - response = self._make_request(params=params) - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'signal': 'sig', - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 3.5, - 'issue': 20200101, - 'lag': 2, - }], - 'message': 'success', - }) - - # make request without specific issue date - params={ - 'source': 'covidcast_nowcast', - 'data_source': 'src', - 'signals': 'sig', - 'sensor_names': 'sensor', - 'time_type': 'day', - 'geo_type': 'county', - 'time_values': 20200101, - 'geo_value': '01001', - } - response = self._make_request(params=params) - - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'signal': 'sig', - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 1.5, - 'issue': 20200103, - 'lag': 2, - }], - 'message': 'success', - }) - - params={ - 'source': 'covidcast_nowcast', - 'data_source': 'src', - 'signals': 'sig', - 'sensor_names': 'sensor', - 'time_type': 'day', - 'geo_type': 'county', - 'time_values': 20200101, - 'geo_value': '01001', - 'as_of': 20200101 - } - response = self._make_request(params=params) - - self.assertEqual(response, { - 'result': 1, - 'epidata': [{ - 'signal': 'sig', - 'time_value': 20200101, - 'geo_value': '01001', - 'value': 3.5, - 'issue': 20200101, - 'lag': 2, - }], - 'message': 'success', - }) diff --git a/src/acquisition/covidcast_nowcast/README.md b/src/acquisition/covidcast_nowcast/README.md deleted file mode 100644 index bc58fb1e2..000000000 --- a/src/acquisition/covidcast_nowcast/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# COVIDCast Nowcasting Sensor Data - -The Nowcasting indicator uses sensorized indicator data to generate and update COVID nowcasts. diff --git a/src/acquisition/covidcast_nowcast/load_sensors.py b/src/acquisition/covidcast_nowcast/load_sensors.py deleted file mode 100644 index 2e2269bb8..000000000 --- a/src/acquisition/covidcast_nowcast/load_sensors.py +++ /dev/null @@ -1,110 +0,0 @@ -from shutil import move -import os -import time - -import pandas as pd -import sqlalchemy - -import delphi.operations.secrets as secrets -from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter, PathDetails - -SENSOR_CSV_PATH = "/common/covidcast_nowcast/receiving/" -SUCCESS_DIR = "archive/successful" -FAIL_DIR = "archive/failed" -TABLE_NAME = "covidcast_nowcast" -DB_NAME = "epidata" -CSV_DTYPES = {"sensor_name": str, "geo_value": str, "value": float} - - -def main(csv_path: str = SENSOR_CSV_PATH) -> None: - """ - Parse all files in a given directory and insert them into the sensor table in the database. - - For all the files found recursively in csv_path that match the naming scheme specified by - CsvImporter.find_csv_files(), attempt to load and insert them into the database. Files which do - not match the naming scheme will be moved to an archive/failed folder and skipped, and files - which raise an error during loading/uploading will be moved to the archive/failed folder and - have the error raised. - - Parameters - ---------- - csv_path - Path to folder containing files to load. - - Returns - ------- - None. - """ - user, pw = secrets.db.epi - engine = sqlalchemy.create_engine(f"mysql+pymysql://{user}:{pw}@{secrets.db.host}/{DB_NAME}") - for filepath, attribute in CsvImporter.find_issue_specific_csv_files(csv_path): - if attribute is None: - _move_after_processing(filepath, success=False) - continue - try: - data = load_and_prepare_file(filepath, attribute) - with engine.connect() as conn: - method = _create_upsert_method(sqlalchemy.MetaData(conn)) - data.to_sql(TABLE_NAME, engine, if_exists="append", method=method, index=False) - except Exception: - _move_after_processing(filepath, success=False) - raise - _move_after_processing(filepath, success=True) - - -def load_and_prepare_file(filepath: str, attributes: PathDetails) -> pd.DataFrame: - """ - Read CSV file into a DataFrame and add relevant attributes as new columns to match DB table. - - Parameters - ---------- - filepath - Path to CSV file. - attributes - (source, signal, time_type, geo_type, time_value, issue, lag) tuple - returned by CsvImport.find_csv_files - - Returns - ------- - DataFrame with additional attributes added as columns based on filename and current date. - """ - data = pd.read_csv(filepath, dtype=CSV_DTYPES) - data["source"] = attributes.source - data["signal"] = attributes.signal - data["time_type"] = attributes.time_type - data["geo_type"] = attributes.geo_type - data["time_value"] = attributes.time_value - data["issue"] = attributes.issue - data["lag"] = attributes.lag - data["value_updated_timestamp"] = int(time.time()) - return data - - -def _move_after_processing(filepath, success): - archive_dir = SUCCESS_DIR if success else FAIL_DIR - new_dir = os.path.dirname(filepath).replace("receiving", archive_dir) - os.makedirs(new_dir, exist_ok=True) - move(filepath, filepath.replace("receiving", archive_dir)) - print(f"{filepath} moved to {archive_dir}") - - -def _create_upsert_method(meta): - def method(table, conn, keys, data_iter): - sql_table = sqlalchemy.Table( - table.name, - meta, - # specify lag column explicitly; lag is a reserved word sqlalchemy doesn't know about - sqlalchemy.Column("lag", sqlalchemy.Integer, quote=True), - autoload=True, - ) - insert_stmt = sqlalchemy.dialects.mysql.insert(sql_table).values( - [dict(zip(keys, data)) for data in data_iter] - ) - upsert_stmt = insert_stmt.on_duplicate_key_update({x.name: x for x in insert_stmt.inserted}) - conn.execute(upsert_stmt) - - return method - - -if __name__ == "__main__": - main() diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R index 90191c95e..d1ea7604b 100644 --- a/src/client/delphi_epidata.R +++ b/src/client/delphi_epidata.R @@ -15,7 +15,7 @@ Epidata <- (function() { # API base url BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/') - client_version <- '4.1.12' + client_version <- '4.1.13' auth <- getOption("epidata.auth", default = NA) diff --git a/src/client/delphi_epidata.d.ts b/src/client/delphi_epidata.d.ts index 0b81db779..03fde6e8c 100644 --- a/src/client/delphi_epidata.d.ts +++ b/src/client/delphi_epidata.d.ts @@ -27,7 +27,6 @@ declare module 'delphi_epidata' { covid_hosp(callback: EpiDataCallback, states: StringParam, dates: EpiRangeParam, issues: EpiRangeParam): Promise; covid_hosp_state_timeseries(callback: EpiDataCallback, states: StringParam, dates: EpiRangeParam, issues: EpiRangeParam): Promise; covidcast_meta(callback: EpiDataCallback): Promise; - covidcast_nowcast(callback: EpiDataCallback, data_source: string, signals: string, time_type: 'day' | 'week', geo_type: string, time_values: EpiRangeParam, as_of?: number, issues?: EpiRangeParam, format?: 'json' | 'tree' | 'classic' | 'csv'): Promise; covidcast(callback: EpiDataCallback, data_source: string, signals: string, time_type: 'day' | 'week', geo_type: string, time_values: EpiRangeParam, as_of?: number, issues?: EpiRangeParam, format?: 'json' | 'tree' | 'classic' | 'csv'): Promise; fluview(callback: EpiDataCallback, regions: StringParam, epiweeks: EpiRangeParam, issues?: EpiRangeParam, lag?: number, auth?: string): Promise; fluview_clinical(callback: EpiDataCallback, regions: StringParam, epiweeks: EpiRangeParam, issues?: EpiRangeParam, lag?: number): Promise; @@ -66,7 +65,6 @@ declare module 'delphi_epidata' { covid_hosp(states: StringParam, dates: EpiRangeParam, issues: EpiRangeParam): Promise; covid_hosp_state_timeseries(states: StringParam, dates: EpiRangeParam, issues: EpiRangeParam): Promise; covidcast_meta(callback: EpiDataCallback): Promise; - covidcast_nowcast(data_source: string, signals: string, time_type: 'day' | 'week', geo_type: string, time_values: EpiRangeParam, as_of?: number, issues?: EpiRangeParam, format?: 'json' | 'tree' | 'classic' | 'csv'): Promise; covidcast(data_source: string, signals: string, time_type: 'day' | 'week', geo_type: string, time_values: EpiRangeParam, as_of?: number, issues?: EpiRangeParam, format?: 'json' | 'tree' | 'classic' | 'csv'): Promise; fluview(regions: StringParam, epiweeks: EpiRangeParam, issues?: EpiRangeParam, lag?: number, auth?: string): Promise; fluview_clinical(regions: StringParam, epiweeks: EpiRangeParam, issues?: EpiRangeParam, lag?: number): Promise; diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js index 395980d55..1ead2aeea 100644 --- a/src/client/delphi_epidata.js +++ b/src/client/delphi_epidata.js @@ -22,7 +22,7 @@ } })(this, function (exports, fetchImpl, jQuery) { const BASE_URL = "https://api.delphi.cmu.edu/epidata/"; - const client_version = "4.1.12"; + const client_version = "4.1.13"; // Helper function to cast values and/or ranges to strings function _listitem(value) { @@ -204,48 +204,6 @@ covidcast_meta: () => { return _request("covidcast_meta", {}); }, - /** - * Fetch Delphi's COVID-19 Surveillance Streams - */ - covidcast_nowcast: ( - data_source, - signals, - time_type, - geo_type, - time_values, - geo_value, - as_of, - issues, - lag, - format - ) => { - requireAll({ - data_source, - signals, - time_type, - geo_type, - time_values, - geo_value, - }); - issuesOrLag(issues, lag); - - const params = { - data_source, - signals, - time_type, - geo_type, - time_values: _list(time_values), - as_of, - issues: _list(issues), - format, - }; - if (Array.isArray(geo_value)) { - params.geo_values = geo_value.join(","); - } else { - params.geo_value = geo_value; - } - return _request("covidcast_nowcast", params); - }, /** * Fetch Delphi's COVID-19 Surveillance Streams */ diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index 99d8f4abc..22052657a 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -70,7 +70,7 @@ def _list(values): @retry(reraise=True, stop=stop_after_attempt(2)) def _request_with_retry(endpoint, params={}): """Make request with a retry if an exception is thrown.""" - request_url = f"{Epidata.BASE_URL}/{endpoint}" + request_url = f"{Epidata.BASE_URL}/{endpoint}/" req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) if req.status_code == 414: req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS) @@ -637,59 +637,6 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c # Make the API call return Epidata._request("covid_hosp_facility_lookup", params) - # Fetch Delphi's COVID-19 Nowcast sensors - @staticmethod - def covidcast_nowcast( - data_source, - signals, - sensor_names, - time_type, - geo_type, - time_values, - geo_value, - as_of=None, - issues=None, - lag=None, - **kwargs, - ): - """Fetch Delphi's COVID-19 Nowcast sensors""" - # Check parameters - # fmt: off - if None in (data_source, signals, time_type, geo_type, time_values, geo_value, sensor_names): - # fmt: on - raise EpidataBadRequestException( - "`data_source`, `signals`, `sensor_names`, `time_type`, `geo_type`, " - "`time_values`, and `geo_value` are all required" - ) - if issues is not None and lag is not None: - raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) - # Set up request - params = { - "data_source": data_source, - "signals": Epidata._list(signals), - "sensor_names": Epidata._list(sensor_names), - "time_type": time_type, - "geo_type": geo_type, - "time_values": Epidata._list(time_values), - } - - if isinstance(geo_value, (list, tuple)): - params["geo_values"] = ",".join(geo_value) - else: - params["geo_value"] = geo_value - if as_of is not None: - params["as_of"] = as_of - if issues is not None: - params["issues"] = Epidata._list(issues) - if lag is not None: - params["lag"] = lag - - if "format" in kwargs: - params["format"] = kwargs["format"] - - # Make the API call - return Epidata._request("covidcast_nowcast", params) - @staticmethod def async_epidata(param_list, batch_size=50): """[DEPRECATED] Make asynchronous Epidata calls for a list of parameters.""" diff --git a/src/client/packaging/npm/package-lock.json b/src/client/packaging/npm/package-lock.json index 17e0054c2..8568c3359 100644 --- a/src/client/packaging/npm/package-lock.json +++ b/src/client/packaging/npm/package-lock.json @@ -1,12 +1,12 @@ { "name": "delphi_epidata", - "version": "4.1.10", + "version": "4.1.12", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "delphi_epidata", - "version": "4.1.10", + "version": "4.1.12", "license": "MIT", "dependencies": { "cross-fetch": "^3.1.4" @@ -17,17 +17,89 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.15.8.tgz", - "integrity": "sha512-2IAnmn8zbvC/jKYhq5Ki9I+DwjlrtMPUCH/CpHvqI4dNnlwHwsxoIhlc8WcYY5LSYknXQtAlFYuHfqAFCvQ4Wg==", + "version": "7.22.13", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", + "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", "dev": true, "dependencies": { - "@babel/highlight": "^7.14.5" + "@babel/highlight": "^7.22.13", + "chalk": "^2.4.2" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@babel/code-frame/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/code-frame/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/code-frame/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/@babel/compat-data": { "version": "7.15.0", "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.15.0.tgz", @@ -77,28 +149,20 @@ } }, "node_modules/@babel/generator": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.15.8.tgz", - "integrity": "sha512-ECmAKstXbp1cvpTTZciZCgfOt6iN64lR0d+euv3UZisU5awfRawOvg07Utn/qBGuH4bRIEZKrA/4LzZyXhZr8g==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", "dev": true, "dependencies": { - "@babel/types": "^7.15.6", - "jsesc": "^2.5.1", - "source-map": "^0.5.0" + "@babel/types": "^7.23.0", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/generator/node_modules/source-map": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/@babel/helper-compilation-targets": { "version": "7.15.4", "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.15.4.tgz", @@ -117,39 +181,35 @@ "@babel/core": "^7.0.0" } }, - "node_modules/@babel/helper-function-name": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.15.4.tgz", - "integrity": "sha512-Z91cOMM4DseLIGOnog+Z8OI6YseR9bua+HpvLAQ2XayUGU+neTtX+97caALaLdyu53I/fjhbeCnWnRH1O3jFOw==", + "node_modules/@babel/helper-environment-visitor": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", "dev": true, - "dependencies": { - "@babel/helper-get-function-arity": "^7.15.4", - "@babel/template": "^7.15.4", - "@babel/types": "^7.15.4" - }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helper-get-function-arity": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.15.4.tgz", - "integrity": "sha512-1/AlxSF92CmGZzHnC515hm4SirTxtpDnLEJ0UyEMgTMZN+6bxXKg04dKhiRx5Enel+SUA1G1t5Ed/yQia0efrA==", + "node_modules/@babel/helper-function-name": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", "dev": true, "dependencies": { - "@babel/types": "^7.15.4" + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-hoist-variables": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.15.4.tgz", - "integrity": "sha512-VTy085egb3jUGVK9ycIxQiPbquesq0HUQ+tPO0uv5mPEBZipk+5FkRKiWq5apuyTE9FUrjENB0rCf8y+n+UuhA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "dependencies": { - "@babel/types": "^7.15.4" + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" @@ -247,21 +307,30 @@ } }, "node_modules/@babel/helper-split-export-declaration": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.15.4.tgz", - "integrity": "sha512-HsFqhLDZ08DxCpBdEVtKmywj6PQbwnF6HHybur0MAnkAKnlS6uHkwnmRIkElB2Owpfb4xL4NwDmDLFubueDXsw==", + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", "dev": true, "dependencies": { - "@babel/types": "^7.15.4" + "@babel/types": "^7.22.5" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.15.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.15.7.tgz", - "integrity": "sha512-K4JvCtQqad9OY2+yTU8w+E82ywk/fe+ELNlt1G8z3bVGlZfn/hOcQQsUhGhW/N+tb3fxK800wLtKOE/aM0m72w==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", "dev": true, "engines": { "node": ">=6.9.0" @@ -291,13 +360,13 @@ } }, "node_modules/@babel/highlight": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz", - "integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", + "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.14.5", - "chalk": "^2.0.0", + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", "js-tokens": "^4.0.0" }, "engines": { @@ -342,13 +411,13 @@ "node_modules/@babel/highlight/node_modules/color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, "node_modules/@babel/highlight/node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true, "engines": { "node": ">=0.8.0" @@ -357,7 +426,7 @@ "node_modules/@babel/highlight/node_modules/has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true, "engines": { "node": ">=4" @@ -376,9 +445,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.15.8.tgz", - "integrity": "sha512-BRYa3wcQnjS/nqI8Ac94pYYpJfojHVvVXJ97+IDCImX4Jc8W8Xv1+47enbruk+q1etOpsQNwnfFcNGw+gtPGxA==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", "dev": true, "bin": { "parser": "bin/babel-parser.js" @@ -550,32 +619,33 @@ } }, "node_modules/@babel/template": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.15.4.tgz", - "integrity": "sha512-UgBAfEa1oGuYgDIPM2G+aHa4Nlo9Lh6mGD2bDBGMTbYnc38vulXPuC1MGjYILIEmlwl6Rd+BPR9ee3gm20CBtg==", + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", + "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.14.5", - "@babel/parser": "^7.15.4", - "@babel/types": "^7.15.4" + "@babel/code-frame": "^7.22.13", + "@babel/parser": "^7.22.15", + "@babel/types": "^7.22.15" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.15.4.tgz", - "integrity": "sha512-W6lQD8l4rUbQR/vYgSuCAE75ADyyQvOpFVsvPPdkhf6lATXAsQIG9YdtOcu8BB1dZ0LKu+Zo3c1wEcbKeuhdlA==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.14.5", - "@babel/generator": "^7.15.4", - "@babel/helper-function-name": "^7.15.4", - "@babel/helper-hoist-variables": "^7.15.4", - "@babel/helper-split-export-declaration": "^7.15.4", - "@babel/parser": "^7.15.4", - "@babel/types": "^7.15.4", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.22.13", + "@babel/generator": "^7.23.0", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", "debug": "^4.1.0", "globals": "^11.1.0" }, @@ -584,12 +654,13 @@ } }, "node_modules/@babel/types": { - "version": "7.15.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.15.6.tgz", - "integrity": "sha512-BPU+7QhqNjmWyDO0/vitH/CuhpV8ZmK1wpKva8nuyNF5MJfuRNWMc+hc14+u9xT93kvykMdncrJT19h74uB1Ig==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", "dev": true, "dependencies": { - "@babel/helper-validator-identifier": "^7.14.9", + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", "to-fast-properties": "^2.0.0" }, "engines": { @@ -867,6 +938,54 @@ "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz", + "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -4088,12 +4207,71 @@ }, "dependencies": { "@babel/code-frame": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.15.8.tgz", - "integrity": "sha512-2IAnmn8zbvC/jKYhq5Ki9I+DwjlrtMPUCH/CpHvqI4dNnlwHwsxoIhlc8WcYY5LSYknXQtAlFYuHfqAFCvQ4Wg==", + "version": "7.22.13", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", + "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", "dev": true, "requires": { - "@babel/highlight": "^7.14.5" + "@babel/highlight": "^7.22.13", + "chalk": "^2.4.2" + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } } }, "@babel/compat-data": { @@ -4134,22 +4312,15 @@ } }, "@babel/generator": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.15.8.tgz", - "integrity": "sha512-ECmAKstXbp1cvpTTZciZCgfOt6iN64lR0d+euv3UZisU5awfRawOvg07Utn/qBGuH4bRIEZKrA/4LzZyXhZr8g==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", "dev": true, "requires": { - "@babel/types": "^7.15.6", - "jsesc": "^2.5.1", - "source-map": "^0.5.0" - }, - "dependencies": { - "source-map": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", - "dev": true - } + "@babel/types": "^7.23.0", + "@jridgewell/gen-mapping": "^0.3.2", + "@jridgewell/trace-mapping": "^0.3.17", + "jsesc": "^2.5.1" } }, "@babel/helper-compilation-targets": { @@ -4164,33 +4335,29 @@ "semver": "^6.3.0" } }, - "@babel/helper-function-name": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.15.4.tgz", - "integrity": "sha512-Z91cOMM4DseLIGOnog+Z8OI6YseR9bua+HpvLAQ2XayUGU+neTtX+97caALaLdyu53I/fjhbeCnWnRH1O3jFOw==", - "dev": true, - "requires": { - "@babel/helper-get-function-arity": "^7.15.4", - "@babel/template": "^7.15.4", - "@babel/types": "^7.15.4" - } + "@babel/helper-environment-visitor": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", + "dev": true }, - "@babel/helper-get-function-arity": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.15.4.tgz", - "integrity": "sha512-1/AlxSF92CmGZzHnC515hm4SirTxtpDnLEJ0UyEMgTMZN+6bxXKg04dKhiRx5Enel+SUA1G1t5Ed/yQia0efrA==", + "@babel/helper-function-name": { + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", "dev": true, "requires": { - "@babel/types": "^7.15.4" + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" } }, "@babel/helper-hoist-variables": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.15.4.tgz", - "integrity": "sha512-VTy085egb3jUGVK9ycIxQiPbquesq0HUQ+tPO0uv5mPEBZipk+5FkRKiWq5apuyTE9FUrjENB0rCf8y+n+UuhA==", + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", + "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", "dev": true, "requires": { - "@babel/types": "^7.15.4" + "@babel/types": "^7.22.5" } }, "@babel/helper-member-expression-to-functions": { @@ -4264,18 +4431,24 @@ } }, "@babel/helper-split-export-declaration": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.15.4.tgz", - "integrity": "sha512-HsFqhLDZ08DxCpBdEVtKmywj6PQbwnF6HHybur0MAnkAKnlS6uHkwnmRIkElB2Owpfb4xL4NwDmDLFubueDXsw==", + "version": "7.22.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", + "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", "dev": true, "requires": { - "@babel/types": "^7.15.4" + "@babel/types": "^7.22.5" } }, + "@babel/helper-string-parser": { + "version": "7.22.5", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", + "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", + "dev": true + }, "@babel/helper-validator-identifier": { - "version": "7.15.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.15.7.tgz", - "integrity": "sha512-K4JvCtQqad9OY2+yTU8w+E82ywk/fe+ELNlt1G8z3bVGlZfn/hOcQQsUhGhW/N+tb3fxK800wLtKOE/aM0m72w==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", "dev": true }, "@babel/helper-validator-option": { @@ -4296,13 +4469,13 @@ } }, "@babel/highlight": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.5.tgz", - "integrity": "sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", + "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.14.5", - "chalk": "^2.0.0", + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", "js-tokens": "^4.0.0" }, "dependencies": { @@ -4338,19 +4511,19 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true }, "supports-color": { @@ -4365,9 +4538,9 @@ } }, "@babel/parser": { - "version": "7.15.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.15.8.tgz", - "integrity": "sha512-BRYa3wcQnjS/nqI8Ac94pYYpJfojHVvVXJ97+IDCImX4Jc8W8Xv1+47enbruk+q1etOpsQNwnfFcNGw+gtPGxA==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", "dev": true }, "@babel/plugin-syntax-async-generators": { @@ -4488,40 +4661,42 @@ } }, "@babel/template": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.15.4.tgz", - "integrity": "sha512-UgBAfEa1oGuYgDIPM2G+aHa4Nlo9Lh6mGD2bDBGMTbYnc38vulXPuC1MGjYILIEmlwl6Rd+BPR9ee3gm20CBtg==", + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", + "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", "dev": true, "requires": { - "@babel/code-frame": "^7.14.5", - "@babel/parser": "^7.15.4", - "@babel/types": "^7.15.4" + "@babel/code-frame": "^7.22.13", + "@babel/parser": "^7.22.15", + "@babel/types": "^7.22.15" } }, "@babel/traverse": { - "version": "7.15.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.15.4.tgz", - "integrity": "sha512-W6lQD8l4rUbQR/vYgSuCAE75ADyyQvOpFVsvPPdkhf6lATXAsQIG9YdtOcu8BB1dZ0LKu+Zo3c1wEcbKeuhdlA==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.14.5", - "@babel/generator": "^7.15.4", - "@babel/helper-function-name": "^7.15.4", - "@babel/helper-hoist-variables": "^7.15.4", - "@babel/helper-split-export-declaration": "^7.15.4", - "@babel/parser": "^7.15.4", - "@babel/types": "^7.15.4", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.22.13", + "@babel/generator": "^7.23.0", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", + "@babel/helper-hoist-variables": "^7.22.5", + "@babel/helper-split-export-declaration": "^7.22.6", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", "debug": "^4.1.0", "globals": "^11.1.0" } }, "@babel/types": { - "version": "7.15.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.15.6.tgz", - "integrity": "sha512-BPU+7QhqNjmWyDO0/vitH/CuhpV8ZmK1wpKva8nuyNF5MJfuRNWMc+hc14+u9xT93kvykMdncrJT19h74uB1Ig==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.14.9", + "@babel/helper-string-parser": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", "to-fast-properties": "^2.0.0" } }, @@ -4741,6 +4916,45 @@ "chalk": "^4.0.0" } }, + "@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", + "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz", + "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json index d1680c4fe..4c1069fd9 100644 --- a/src/client/packaging/npm/package.json +++ b/src/client/packaging/npm/package.json @@ -2,7 +2,7 @@ "name": "delphi_epidata", "description": "Delphi Epidata API Client", "authors": "Delphi Group", - "version": "4.1.12", + "version": "4.1.13", "license": "MIT", "homepage": "https://github.com/cmu-delphi/delphi-epidata", "bugs": { diff --git a/src/client/packaging/pypi/delphi_epidata/__init__.py b/src/client/packaging/pypi/delphi_epidata/__init__.py index 5115c39ef..1628aeb61 100644 --- a/src/client/packaging/pypi/delphi_epidata/__init__.py +++ b/src/client/packaging/pypi/delphi_epidata/__init__.py @@ -1,4 +1,4 @@ from .delphi_epidata import Epidata name = "delphi_epidata" -__version__ = "4.1.12" +__version__ = "4.1.13" diff --git a/src/client/packaging/pypi/setup.py b/src/client/packaging/pypi/setup.py index a0aefc954..13b2441d6 100644 --- a/src/client/packaging/pypi/setup.py +++ b/src/client/packaging/pypi/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="delphi_epidata", - version="4.1.12", + version="4.1.13", author="David Farrow", author_email="dfarrow0@gmail.com", description="A programmatic interface to Delphi's Epidata API.", diff --git a/src/ddl/covidcast_nowcast.sql b/src/ddl/covidcast_nowcast.sql deleted file mode 100644 index 8c3944f6e..000000000 --- a/src/ddl/covidcast_nowcast.sql +++ /dev/null @@ -1,77 +0,0 @@ -USE epidata; -/* -This table stores various sensors of Delphi's COVID-19 surveillance streams for nowcasting. -*/ - -/* -`covidcast_nowcast` stores daily sensor readings at various geographic resolutions. - -Data is not intended for public consumption at the time and is undocumented, but the endpoint is public. - -+------------------------------+-------------+------+-----+---------+----------------+ -| Field | Type | Null | Key | Default | Extra | -+------------------------------+-------------+------+-----+---------+----------------+ -| id | int(11) | NO | PRI | NULL | auto_increment | -| source | varchar(32) | NO | MUL | NULL | | -| signal | varchar(64) | NO | | NULL | | -| sensor_name | varchar(64) | NO | | NULL | | -| time_type | varchar(12) | NO | | NULL | | -| geo_type | varchar(12) | NO | | NULL | | -| time_value | int(11) | NO | | NULL | | -| geo_value | varchar(12) | NO | | NULL | | -| value_updated_timestamp | int(11) | NO | | NULL | | -| value | double | NO | | NULL | | -| issue | int(11) | NO | | NULL | | -| lag | int(11) | NO | | NULL | | -+------------------------------+-------------+------+-----+---------+----------------+ - -- `id` - unique identifier for each record -- `source` - name of upstream data souce -- `signal` - name of signal derived from upstream data -- `sensor_name` - name of sensor derived from specific source and signal. -- `time_type` - temporal resolution of the signal (e.g. day, week) -- `geo_type` - spatial resolution of the signal (e.g. county, HRR, MSA, DMA, state) -- `time_value` - time unit (e.g. date) over which underlying events happened -- `geo_value` - a unique code for each location, depending on `geo_type` - - county: FIPS 6-4 code - - nation: 2 letter nation code - - state: two-letter state abbreviation -- `value_updated_timestamp` - time when primary data (e.g. `value`) was last updated -- `value` - sensor value derived from underlying indicator -- `issue` - the time_value of publication -- `lag` - the number of time_type units between `time_value` and `issue` -*/ - -CREATE TABLE `covidcast_nowcast` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `source` varchar(32) NOT NULL, - `signal` varchar(64) NOT NULL, - `sensor_name` varchar(64) NOT NULL, - `time_type` varchar(12) NOT NULL, - `geo_type` varchar(12) NOT NULL, - `time_value` int(11) NOT NULL, - `geo_value` varchar(12) NOT NULL, - -- "primary" values are derived from the upstream data source - `value_updated_timestamp` int(11) NOT NULL, - `value` double NOT NULL, - `issue` int(11) NOT NULL, - `lag` int(11) NOT NULL, - PRIMARY KEY (`id`), - -- for uniqueness, and also fast lookup of all locations on a given date - UNIQUE KEY (`source`, `signal`, `sensor_name`, `time_type`, `geo_type`, `time_value`, `geo_value`, `issue`), - -- for fast lookup of a time-series for a given location - KEY `by_issue` (`source`, `signal`, `sensor_name`, `time_type`, `geo_type`, `geo_value`, `time_value`, `issue`), - KEY `by_lag` (`source`, `signal`, `sensor_name`, `time_type`, `geo_type`, `geo_value`, `time_value`, `lag`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/src/server/_config.py b/src/server/_config.py index c6eaedaf4..caed3ffb9 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -7,7 +7,7 @@ load_dotenv() -VERSION = "4.1.12" +VERSION = "4.1.13" MAX_RESULTS = int(10e6) MAX_COMPATIBILITY_RESULTS = int(3650) diff --git a/src/server/endpoints/__init__.py b/src/server/endpoints/__init__.py index 94f1de5b8..6dad05bf3 100644 --- a/src/server/endpoints/__init__.py +++ b/src/server/endpoints/__init__.py @@ -3,7 +3,6 @@ covid_hosp_facility_lookup, covid_hosp_facility, covid_hosp_state_timeseries, - covidcast_nowcast, covidcast_meta, covidcast, delphi, @@ -38,7 +37,6 @@ covid_hosp_facility_lookup, covid_hosp_facility, covid_hosp_state_timeseries, - covidcast_nowcast, covidcast_meta, covidcast, delphi, diff --git a/src/server/endpoints/covidcast_nowcast.py b/src/server/endpoints/covidcast_nowcast.py deleted file mode 100644 index 79cb71670..000000000 --- a/src/server/endpoints/covidcast_nowcast.py +++ /dev/null @@ -1,105 +0,0 @@ -from flask import Blueprint, request - -from .._params import ( - extract_date, - extract_dates, - extract_integer, - extract_strings -) -from .._query import execute_query, filter_integers, filter_strings -from .._validate import ( - require_all, - require_any, -) - -# first argument is the endpoint name -bp = Blueprint("covidcast_nowcast", __name__) -alias = None - - -@bp.route("/", methods=("GET", "POST")) -def handle(): - require_all(request, - "data_source", "time_type", "geo_type", "time_values", "signals", "sensor_names" - ) - require_any(request, "geo_value", "geo_values", empty=True) - - time_values = extract_dates("time_values") - as_of = extract_date("as_of") - issues = extract_dates("issues") - lag = extract_integer("lag") - signals = extract_strings(("signals", "signal")) - sensor_names = extract_strings("sensor_names") - geo_values = extract_strings(("geo_values", "geo_value")) - data_source = request.values["data_source"] - time_type = request.values["time_type"] - geo_type = request.values["geo_type"] - - # build query - table = "`covidcast_nowcast` t" - fields = "t.`signal`, t.`time_value`, t.`geo_value`, t.`value`, t.`issue`, t.`lag`" - order = "t.`signal` ASC, t.`time_value` ASC, t.`geo_value` ASC, t.`issue` ASC" - - params = dict() - # basic query info - # data type of each field - # build the source, signal, time, and location (type and id) filters - condition_source = "t.`source` = :source" - params["source"] = data_source - condition_signal = filter_strings("t.`signal`", signals, "signal", params) - condition_sensor_name = filter_strings( - "t.`sensor_name`", sensor_names, "sensor_name", params - ) - condition_time_type = "t.`time_type` = :time_type" - params["time_type"] = time_type - condition_geo_type = "t.`geo_type` = :geo_type" - params["geo_type"] = geo_type - condition_time_value = filter_integers( - "t.`time_value`", time_values, "time_value", params - ) - - if not geo_values: - condition_geo_value = "FALSE" - elif len(geo_values) == 1 and geo_values[0] == "*": - # the wildcard query should return data for all locations in `geo_type` - condition_geo_value = "TRUE" - else: - # return data for multiple location - condition_geo_value = filter_strings( - "t.`geo_value`", geo_values, "geo_value", params - ) - - conditions = f"({condition_source}) AND ({condition_signal}) AND ({condition_sensor_name}) AND ({condition_time_type}) AND ({condition_geo_type}) AND ({condition_time_value}) AND ({condition_geo_value})" - - subquery = "" - if issues: - # build the issue filter - condition_issue = filter_integers("t.`issue`", issues, "issue", params) - query = f"SELECT {fields} FROM {table} {subquery} WHERE {conditions} AND ({condition_issue}) ORDER BY {order}" - elif lag: - # build the lag filter - condition_lag = "(t.`lag` = :lag)" - params["lag"] = lag - query = f"SELECT {fields} FROM {table} {subquery} WHERE {conditions} AND ({condition_lag}) ORDER BY {order}" - elif as_of: - # fetch most recent issues with as of - sub_condition_asof = "(`issue` <= :as_of)" - params["as_of"] = as_of - sub_fields = "max(`issue`) `max_issue`, `time_type`, `time_value`, `source`, `signal`, `geo_type`, `geo_value`" - sub_group = ( - "`time_type`, `time_value`, `source`, `signal`, `geo_type`, `geo_value`" - ) - sub_condition = "x.`max_issue` = t.`issue` AND x.`time_type` = t.`time_type` AND x.`time_value` = t.`time_value` AND x.`source` = t.`source` AND x.`signal` = t.`signal` AND x.`geo_type` = t.`geo_type` AND x.`geo_value` = t.`geo_value`" - subquery = f"JOIN (SELECT {sub_fields} FROM {table} WHERE ({conditions} AND {sub_condition_asof}) GROUP BY {sub_group}) x ON {sub_condition}" - condition_version = "TRUE" - query = f"SELECT {fields} FROM {table} {subquery} WHERE {conditions} AND ({condition_version}) ORDER BY {order}" - else: - # fetch most recent issue fast - query = f"WITH t as (SELECT {fields}, ROW_NUMBER() OVER (PARTITION BY t.`time_type`, t.`time_value`, t.`source`, t.`signal`, t.`geo_type`, t.`geo_value` ORDER BY t.`issue` DESC) `row` FROM {table} {subquery} WHERE {conditions}) SELECT {fields} FROM t where `row` = 1 ORDER BY {order}" - - fields_string = ["geo_value", "signal"] - fields_int = ["time_value", "issue", "lag"] - fields_float = ["value"] - - # send query - return execute_query(query, params, fields_string, fields_int, fields_float) diff --git a/src/server/endpoints/covidcast_utils/dashboard_signals.py b/src/server/endpoints/covidcast_utils/dashboard_signals.py index 42f3d612d..9bd564575 100644 --- a/src/server/endpoints/covidcast_utils/dashboard_signals.py +++ b/src/server/endpoints/covidcast_utils/dashboard_signals.py @@ -30,6 +30,10 @@ def __new__(cls): with open(module_dir / 'descriptions.raw.txt', 'r') as f: for desc in yaml.safe_load_all(f): srcsigs.append( (desc['Id'], desc['Signal']) ) + if 'Overrides' in desc: + for sub_geo in desc['Overrides']: + sub_signal = desc['Overrides'][sub_geo] + srcsigs.append( (sub_signal['Id'], sub_signal['Signal']) ) source_id = None with open(module_dir / 'questions.raw.txt', 'r') as f: diff --git a/src/server/endpoints/covidcast_utils/descriptions.raw.txt b/src/server/endpoints/covidcast_utils/descriptions.raw.txt index e0d057aab..98bda8f1e 100644 --- a/src/server/endpoints/covidcast_utils/descriptions.raw.txt +++ b/src/server/endpoints/covidcast_utils/descriptions.raw.txt @@ -43,18 +43,6 @@ SignalTooltip: Percentage of daily doctor visits that are due to lab-confirmed i Description: Delphi receives aggregated statistics from Change Healthcare, Inc. on lab-confirmed influenza outpatient doctor visits, derived from ICD codes found in insurance claims. Using this data, we estimate the percentage of daily doctor’s visits in each area that resulted in a diagnosis of influenza. Note that these estimates are based only on visits by patients whose insurance claims are accessible to Change Healthcare. --- -Name: COVID Cases -Id: jhu-csse -Signal: confirmed_7dav_incidence_prop -Highlight: [default] -ExtendedColorScale: true - - -SignalTooltip: Newly reported COVID-19 cases per 100,000 people, based on data from Johns Hopkins University - - -Description: This data shows the number of COVID-19 confirmed cases newly reported each day. It reflects only cases reported by state and local health authorities. It is based on case counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/). The signal may not be directly comparable across regions with vastly different testing capacity or reporting criteria. ---- Name: COVID Hospital Admissions Id: hhs Signal: confirmed_admissions_covid_1d_prop_7dav @@ -63,8 +51,8 @@ ExtendedColorScale: true Levels: [nation, state, county] Overrides: County: - Id: dsew-cpr - Signal: confirmed_admissions_covid_1d_prop_7dav + Id: hospital-admissions + Signal: smoothed_adj_covid19_from_claims @@ -89,11 +77,11 @@ SignalTooltip: Confirmed influenza hospital admissions per 100,000 people Description: This data shows the number of hospital admissions with lab-confirmed influenza each day. We source this data from the Report on Patient Impact and Hospital Capacity published by the US Department of Health & Human Services (HHS). --- Name: COVID Deaths -Id: jhu-csse -Signal: deaths_7dav_incidence_prop +Id: nchs-mortality +Signal: deaths_covid_incidence_prop -SignalTooltip: Newly reported COVID-19 deaths per 100,000 people, based on data from Johns Hopkins University +SignalTooltip: Newly reported COVID-19 deaths per 100,000 people, based on NCHS mortality data. -Description: This data shows the number of COVID-19 deaths newly reported each day. The signal is based on COVID-19 death counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/). \ No newline at end of file +Description: This data shows the number of COVID-19 deaths newly reported each week. The signal is based on COVID-19 death counts compiled and made public by [the National Center for Health Statistics](https://www.cdc.gov/nchs/nvss/vsrr/COVID19/index.htm). diff --git a/tests/acquisition/covidcast_nowcast/__init__.py b/tests/acquisition/covidcast_nowcast/__init__.py deleted file mode 100644 index e197f3ec4..000000000 --- a/tests/acquisition/covidcast_nowcast/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -import sys -import os - -sys.path.append(os.getcwd()) diff --git a/tests/acquisition/covidcast_nowcast/test_load_sensors.py b/tests/acquisition/covidcast_nowcast/test_load_sensors.py deleted file mode 100644 index efa21bcc7..000000000 --- a/tests/acquisition/covidcast_nowcast/test_load_sensors.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Unit tests for update.py.""" - -# standard library -import unittest -from unittest import mock -from io import StringIO - -# third party -import pandas as pd - -# first party -from delphi.epidata.acquisition.covidcast.csv_importer import PathDetails -from delphi.epidata.acquisition.covidcast_nowcast.load_sensors import main, load_and_prepare_file - -# py3tester coverage target -__test_target__ = "delphi.epidata.acquisition.covidcast_nowcast.load_sensors" - - -class UpdateTests(unittest.TestCase): - @mock.patch("time.time", mock.MagicMock(return_value=12345)) - def test_load_and_prepare_file(self): - - test_attributes = PathDetails( - 20210102, - 3, - "test_source", - "test_signal", - "test_time_type", - 20201231, - "test_geo_type", - ) - - test_df = load_and_prepare_file( - StringIO("sensor_name,geo_value,value\ntestname,01001,1.5"), test_attributes - ) - pd.testing.assert_frame_equal( - test_df, - pd.DataFrame( - { - "sensor_name": ["testname"], - "geo_value": ["01001"], - "value": [1.5], - "source": ["test_source"], - "signal": ["test_signal"], - "time_type": ["test_time_type"], - "geo_type": ["test_geo_type"], - "time_value": [20201231], - "issue": [20210102], - "lag": [3], - "value_updated_timestamp": [12345], - } - ), - )