From d695b2378a0d8dd5dda1fcb7bbcda73affaf2d13 Mon Sep 17 00:00:00 2001 From: msoltadeo Date: Tue, 17 Mar 2020 11:13:53 -0300 Subject: [PATCH 01/46] updates for new pandas version --- urbansim/models/dcm.py | 8 ++++---- urbansim/urbanchoice/tests/test_mnl.py | 14 +++++++------- urbansim/utils/misc.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/urbansim/models/dcm.py b/urbansim/models/dcm.py index b649ad1d..fc62eeeb 100644 --- a/urbansim/models/dcm.py +++ b/urbansim/models/dcm.py @@ -413,14 +413,14 @@ def fit(self, choosers, alternatives, current_choice): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Estimated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' 'the input columns.') self.log_likelihoods, self.fit_parameters = mnl.mnl_estimate( - model_design.as_matrix(), chosen, self.sample_size) + model_design.values, chosen, self.sample_size) self.fit_parameters.index = model_design.columns logger.debug('finish: fit LCM model {}'.format(self.name)) @@ -523,7 +523,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): model_design = dmatrix( self.str_model_expression, data=merged, return_type='dataframe') - if len(merged) != model_design.as_matrix().shape[0]: + if len(merged) != model_design.values.shape[0]: raise ModelEvaluationError( 'Simulated data does not have the same length as input. ' 'This suggests there are null values in one or more of ' @@ -542,7 +542,7 @@ def probabilities(self, choosers, alternatives, filter_tables=True): numalts = sample_size probabilities = mnl.mnl_simulate( - model_design.as_matrix(), + model_design.values, coeffs, numalts=numalts, returnprobs=True) diff --git a/urbansim/urbanchoice/tests/test_mnl.py b/urbansim/urbanchoice/tests/test_mnl.py index a939e462..d9a067d5 100644 --- a/urbansim/urbanchoice/tests/test_mnl.py +++ b/urbansim/urbanchoice/tests/test_mnl.py @@ -110,12 +110,12 @@ def choosers_dm(choosers, test_data): @pytest.fixture def fit_coeffs(dm, chosen, num_alts): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) return fit.Coefficient.values def test_mnl_estimate(dm, chosen, num_alts, test_data): - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(test_data['est_expected']) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -134,10 +134,10 @@ def test_mnl_simulate(dm, fit_coeffs, num_alts, test_data, choosers_dm): # now test with real data probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=test_data['sim_expected'].columns) results, expected = results.align(test_data['sim_expected']) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values), expected.values), rtol=1e-4) def test_alternative_specific_coeffs(num_alts): @@ -193,7 +193,7 @@ def test_alternative_specific_coeffs(num_alts): 'boat:(intercept)', 'charter:(intercept)', 'pier:(intercept)', 'boat:income', 'charter:income', 'pier:income']) - log_like, fit = mnl.mnl_estimate(dm.as_matrix(), fish_chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values), fish_chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(expected) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -206,7 +206,7 @@ def test_alternative_specific_coeffs(num_alts): fit_coeffs = fit.Coefficient.values probs = mnl.mnl_simulate( - choosers_dm.as_matrix(), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=expected.columns) results, expected = results.align(expected) - npt.assert_allclose(results.as_matrix(), expected.as_matrix(), rtol=1e-4) + npt.assert_allclose(results.values), expected.values), rtol=1e-4) diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index b2aadfd3..3d7b8350 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -138,7 +138,7 @@ def compute_range(travel_data, attr, travel_time_attr, dist, agg=np.sum): """ travel_data = travel_data.reset_index(level=1) travel_data = travel_data[travel_data[travel_time_attr] < dist] - travel_data["attr"] = attr[travel_data.to_zone_id].values + travel_data["attr"] = attr.reindex(travel_data.to_zone_id, fill_value=0).values return travel_data.groupby(level=0).attr.apply(agg) From 9c43d04702f1c2580d075ebb5619eb31a3700d68 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 11:03:59 -0700 Subject: [PATCH 02/46] Fixing typos in search-and-replace --- urbansim/urbanchoice/tests/test_mnl.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/urbansim/urbanchoice/tests/test_mnl.py b/urbansim/urbanchoice/tests/test_mnl.py index d9a067d5..ff6ef4cf 100644 --- a/urbansim/urbanchoice/tests/test_mnl.py +++ b/urbansim/urbanchoice/tests/test_mnl.py @@ -110,12 +110,12 @@ def choosers_dm(choosers, test_data): @pytest.fixture def fit_coeffs(dm, chosen, num_alts): - log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) return fit.Coefficient.values def test_mnl_estimate(dm, chosen, num_alts, test_data): - log_like, fit = mnl.mnl_estimate(dm.values), chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(test_data['est_expected']) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -134,10 +134,10 @@ def test_mnl_simulate(dm, fit_coeffs, num_alts, test_data, choosers_dm): # now test with real data probs = mnl.mnl_simulate( - choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=test_data['sim_expected'].columns) results, expected = results.align(test_data['sim_expected']) - npt.assert_allclose(results.values), expected.values), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) def test_alternative_specific_coeffs(num_alts): @@ -193,7 +193,7 @@ def test_alternative_specific_coeffs(num_alts): 'boat:(intercept)', 'charter:(intercept)', 'pier:(intercept)', 'boat:income', 'charter:income', 'pier:income']) - log_like, fit = mnl.mnl_estimate(dm.values), fish_chosen, num_alts) + log_like, fit = mnl.mnl_estimate(dm.values, fish_chosen, num_alts) result = pd.Series(fit.Coefficient.values, index=dm.columns) result, expected = result.align(expected) npt.assert_allclose(result.values, expected.values, rtol=1e-4) @@ -206,7 +206,7 @@ def test_alternative_specific_coeffs(num_alts): fit_coeffs = fit.Coefficient.values probs = mnl.mnl_simulate( - choosers_dm.values), fit_coeffs, num_alts, returnprobs=True) + choosers_dm.values, fit_coeffs, num_alts, returnprobs=True) results = pd.DataFrame(probs, columns=expected.columns) results, expected = results.align(expected) - npt.assert_allclose(results.values), expected.values), rtol=1e-4) + npt.assert_allclose(results.values, expected.values, rtol=1e-4) From 61b1d50e74deff96d8cdf83c7b20ad564735190f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 14:33:35 -0700 Subject: [PATCH 03/46] Fixing column list comparison --- urbansim/urbanchoice/tests/test_interaction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urbansim/urbanchoice/tests/test_interaction.py b/urbansim/urbanchoice/tests/test_interaction.py index 938edcf4..aeca68a5 100644 --- a/urbansim/urbanchoice/tests/test_interaction.py +++ b/urbansim/urbanchoice/tests/test_interaction.py @@ -35,8 +35,8 @@ def test_interaction_dataset_sim(choosers, alternatives): assert len(merged) == len(choosers) * len(alternatives) npt.assert_array_equal(merged.index.values, sample) - assert list(merged.columns) == [ - 'var2', 'var3', 'join_index', 'thing_id', 'var1'] + assert set(list(merged.columns)) == set([ + 'var2', 'var3', 'join_index', 'thing_id', 'var1']) npt.assert_array_equal( merged['var1'].values, choosers['var1'].values.repeat(len(alternatives))) From 2954eb2e11a447caa1eeda4d21139e398ec81364 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 14:38:24 -0700 Subject: [PATCH 04/46] .ix[] -> .loc[] --- urbansim/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index 3d7b8350..e758a560 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -357,7 +357,7 @@ def series64bitto32bit(s): def _pandassummarytojson(v, ndigits=3): - return {i: round(float(v.ix[i]), ndigits) for i in v.index} + return {i: round(float(v.loc[i]), ndigits) for i in v.index} def pandasdfsummarytojson(df, ndigits=3): From 893732e5b5cce86f54f6897e75dcc6fdc13924f9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:06:06 -0700 Subject: [PATCH 05/46] Updating travis environment --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15b74bbe..55d656c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,8 @@ language: python sudo: false python: - '2.7' -- '3.5' +- '3.6' +- '3.8' install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh @@ -16,11 +17,11 @@ install: - > conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy - statsmodels pytables pytest pyyaml toolz + cytoolz jupyter jinja2 matplotlib numpy pandas pandana patsy pip scipy + statsmodels pytables pytest pyyaml toolz prettytable - source activate test-environment - pip install orca osmnet pandana bottle simplejson zbox -- pip install pytest-cov coveralls pycodestyle +- pip install 'pytest<4.0' pytest-cov coveralls pycodestyle - pip install . before_script: - git clone https://github.com/udst/sanfran_urbansim.git From f698273c6f70a438700400e650e1ccb5471f26d5 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:17:26 -0700 Subject: [PATCH 06/46] Travis environment adjustments --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55d656c4..0486f6fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,11 @@ install: - > conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - cytoolz jupyter jinja2 matplotlib numpy pandas pandana patsy pip scipy - statsmodels pytables pytest pyyaml toolz prettytable + cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy + statsmodels pytables pyyaml toolz - source activate test-environment -- pip install orca osmnet pandana bottle simplejson zbox +- conda install pandana -c conda-forge +- pip install orca osmnet pandana bottle simplejson zbox prettytable - pip install 'pytest<4.0' pytest-cov coveralls pycodestyle - pip install . before_script: From 016ff41c2a7930062ef1d1b595f8f777765a3005 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:25:09 -0700 Subject: [PATCH 07/46] Updating appveyor environment --- appveyor.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 075cdeca..b1dbde08 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ environment: matrix: - PYTHON_VERSION: 2.7 MINICONDA: C:\Miniconda - - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 3.6 MINICONDA: C:\Miniconda3 init: @@ -15,11 +15,11 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz ipython-notebook jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pytest pyyaml toolz" + - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas - - pip install orca osmnet pandana bottle simplejson zbox - - pip install pycodestyle + - conda install -c conda-forge shapely geopandas pandana + - pip install orca osmnet bottle simplejson zbox prettytable + - pip install 'pytest<4.0' pycodestyle - pip install . before_test: From 3200e8acc5ceb915abbe4b6e9798de3f48975880 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:37:34 -0700 Subject: [PATCH 08/46] Switching pandana from conda to pip for appveyor --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b1dbde08..e7776993 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,8 +17,8 @@ install: - conda info -a - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas pandana - - pip install orca osmnet bottle simplejson zbox prettytable + - conda install -c conda-forge shapely geopandas + - pip install orca pandana osmnet bottle simplejson zbox prettytable - pip install 'pytest<4.0' pycodestyle - pip install . From 6c2cbb044a6eec9731bc59a860de842436c42138 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 15:59:23 -0700 Subject: [PATCH 09/46] AppVeyor syntax --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index e7776993..df40f72a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ install: - activate test-environment - conda install -c conda-forge shapely geopandas - pip install orca pandana osmnet bottle simplejson zbox prettytable - - pip install 'pytest<4.0' pycodestyle + - pip install pytest<4.0 pycodestyle - pip install . before_test: From a437a81f2328bd692289615fba1ead3f28e97166 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 18 Mar 2020 17:53:10 -0700 Subject: [PATCH 10/46] Another try --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index df40f72a..e9d0bb7e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ install: - activate test-environment - conda install -c conda-forge shapely geopandas - pip install orca pandana osmnet bottle simplejson zbox prettytable - - pip install pytest<4.0 pycodestyle + - "pip install pytest<4.0 pycodestyle" - pip install . before_test: From effd3a8df22c91731ac46c6f6cb7ed3357cc7877 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 19 Mar 2020 21:12:40 -0700 Subject: [PATCH 11/46] Pytest pip -> conda for appveyor --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e9d0bb7e..0ab7faf2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,9 +17,9 @@ install: - conda info -a - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - activate test-environment - - conda install -c conda-forge shapely geopandas + - conda install -c conda-forge shapely geopandas pytest=3.10 - pip install orca pandana osmnet bottle simplejson zbox prettytable - - "pip install pytest<4.0 pycodestyle" + - pip install pycodestyle - pip install . before_test: From c9b49115253bfe7f28d53a8de263cc9e6bdcff18 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 23 Mar 2020 14:24:40 -0700 Subject: [PATCH 12/46] Revising tests that now fail in win py27 --- urbansim/utils/tests/test_testing.py | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/urbansim/utils/tests/test_testing.py b/urbansim/utils/tests/test_testing.py index 642e8a56..f2ad800f 100644 --- a/urbansim/utils/tests/test_testing.py +++ b/urbansim/utils/tests/test_testing.py @@ -17,36 +17,36 @@ def test_frames_equal_mismatched_columns(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'b': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected column 'a' not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_rows(): expected = pd.DataFrame({'a': [1]}, index=[0]) actual = pd.DataFrame({'a': [1]}, index=[1]) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == "Expected row 0 not found." + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal_mismatched_items(): expected = pd.DataFrame({'a': [1]}) actual = pd.DataFrame({'a': [2]}) - with pytest.raises(AssertionError) as info: + try: testing.assert_frames_equal(actual, expected) - - assert str(info.value) == """ -Items are not equal: - ACTUAL: 2 - DESIRED: 1 - -Column: 'a' -Row: 0""" + except AssertionError: + pass + else: + raise AssertionError def test_frames_equal(): From 44ca1d47e50fc9d6e12048329a6ff8907fbbf381 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 22 Apr 2020 15:40:08 -0700 Subject: [PATCH 13/46] yaml.load -> yaml.safe_load --- urbansim/models/tests/test_dcm.py | 12 ++++++------ urbansim/models/tests/test_regression.py | 12 ++++++------ urbansim/utils/networks.py | 2 +- urbansim/utils/tests/test_yamlio.py | 6 +++--- urbansim/utils/yamlio.py | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/urbansim/models/tests/test_dcm.py b/urbansim/models/tests/test_dcm.py index fa60ba06..e1254dba 100644 --- a/urbansim/models/tests/test_dcm.py +++ b/urbansim/models/tests/test_dcm.py @@ -222,10 +222,10 @@ def test_mnl_dcm_yaml(basic_dcm, choosers, alternatives): 'fit_parameters': None } - assert yaml.load(basic_dcm.to_yaml()) == expected_dict + assert yaml.safe_load(basic_dcm.to_yaml()) == expected_dict new_mod = dcm.MNLDiscreteChoiceModel.from_yaml(basic_dcm.to_yaml()) - assert yaml.load(new_mod.to_yaml()) == expected_dict + assert yaml.safe_load(new_mod.to_yaml()) == expected_dict basic_dcm.fit(choosers, alternatives, 'thing_id') @@ -233,7 +233,7 @@ def test_mnl_dcm_yaml(basic_dcm, choosers, alternatives): del expected_dict['log_likelihoods'] del expected_dict['fit_parameters'] - actual_dict = yaml.load(basic_dcm.to_yaml()) + actual_dict = yaml.safe_load(basic_dcm.to_yaml()) assert isinstance(actual_dict.pop('log_likelihoods'), dict) assert isinstance(actual_dict.pop('fit_parameters'), dict) @@ -524,10 +524,10 @@ def test_mnl_dcm_segmented_yaml(grouped_choosers, alternatives): } } - assert yaml.load(group.to_yaml()) == expected_dict + assert yaml.safe_load(group.to_yaml()) == expected_dict new_seg = dcm.SegmentedMNLDiscreteChoiceModel.from_yaml(group.to_yaml()) - assert yaml.load(new_seg.to_yaml()) == expected_dict + assert yaml.safe_load(new_seg.to_yaml()) == expected_dict group.fit(grouped_choosers, alternatives, 'thing_id') @@ -539,7 +539,7 @@ def test_mnl_dcm_segmented_yaml(grouped_choosers, alternatives): del expected_dict['models']['y']['fit_parameters'] del expected_dict['models']['y']['log_likelihoods'] - actual_dict = yaml.load(group.to_yaml()) + actual_dict = yaml.safe_load(group.to_yaml()) assert isinstance(actual_dict['models']['x'].pop('fit_parameters'), dict) assert isinstance(actual_dict['models']['x'].pop('log_likelihoods'), dict) assert isinstance(actual_dict['models']['y'].pop('fit_parameters'), dict) diff --git a/urbansim/models/tests/test_regression.py b/urbansim/models/tests/test_regression.py index bc55ad58..de1743f2 100644 --- a/urbansim/models/tests/test_regression.py +++ b/urbansim/models/tests/test_regression.py @@ -211,7 +211,7 @@ def setup_method(self, method): def test_string(self): test_yaml = self.model.to_yaml() - assert_dict_specs_equal(yaml.load(test_yaml), self.expected_dict) + assert_dict_specs_equal(yaml.safe_load(test_yaml), self.expected_dict) model = regression.RegressionModel.from_yaml(yaml_str=test_yaml) assert isinstance(model, regression.RegressionModel) @@ -220,7 +220,7 @@ def test_buffer(self): test_buffer = StringIO() self.model.to_yaml(str_or_buffer=test_buffer) assert_dict_specs_equal( - yaml.load(test_buffer.getvalue()), self.expected_dict) + yaml.safe_load(test_buffer.getvalue()), self.expected_dict) test_buffer.seek(0) model = regression.RegressionModel.from_yaml(str_or_buffer=test_buffer) @@ -233,7 +233,7 @@ def test_file(self): self.model.to_yaml(str_or_buffer=test_file) with open(test_file) as f: - assert_dict_specs_equal(yaml.load(f), self.expected_dict) + assert_dict_specs_equal(yaml.safe_load(f), self.expected_dict) model = regression.RegressionModel.from_yaml(str_or_buffer=test_file) assert isinstance(model, regression.RegressionModel) @@ -375,10 +375,10 @@ def test_SegmentedRegressionModel_yaml(groupby_df): } } - assert yaml.load(seg.to_yaml()) == expected_dict + assert yaml.safe_load(seg.to_yaml()) == expected_dict new_seg = regression.SegmentedRegressionModel.from_yaml(seg.to_yaml()) - assert yaml.load(new_seg.to_yaml()) == expected_dict + assert yaml.safe_load(new_seg.to_yaml()) == expected_dict seg.fit(groupby_df) @@ -392,7 +392,7 @@ def test_SegmentedRegressionModel_yaml(groupby_df): del expected_dict['models']['y']['fit_rsquared'] del expected_dict['models']['y']['fit_rsquared_adj'] - actual_dict = yaml.load(seg.to_yaml()) + actual_dict = yaml.safe_load(seg.to_yaml()) assert isinstance(actual_dict['models']['x'].pop('fit_parameters'), dict) assert isinstance(actual_dict['models']['x'].pop('fit_rsquared'), float) assert isinstance( diff --git a/urbansim/utils/networks.py b/urbansim/utils/networks.py index 00549ffb..5e4757e8 100644 --- a/urbansim/utils/networks.py +++ b/urbansim/utils/networks.py @@ -15,7 +15,7 @@ def from_yaml(net, cfgname): print("Computing accessibility variables") - cfg = yaml.load(open(misc.config(cfgname))) + cfg = yaml.safe_load(open(misc.config(cfgname))) nodes = pd.DataFrame(index=net.node_ids) diff --git a/urbansim/utils/tests/test_yamlio.py b/urbansim/utils/tests/test_yamlio.py index d5804bed..f17cdecc 100644 --- a/urbansim/utils/tests/test_yamlio.py +++ b/urbansim/utils/tests/test_yamlio.py @@ -123,7 +123,7 @@ def test_series_to_yaml_safe_int_index(): assert d == {0: 100, 1: 101, 2: 102} y = yaml.dump(d, default_flow_style=False) - assert_series_equal(pd.Series(yaml.load(y)), s) + assert_series_equal(pd.Series(yaml.safe_load(y)), s) def test_series_to_yaml_safe_str_index(): @@ -133,7 +133,7 @@ def test_series_to_yaml_safe_str_index(): assert d == {'x': 'a', 'y': 'b', 'z': 'c'} y = yaml.dump(d, default_flow_style=False) - assert_series_equal(pd.Series(yaml.load(y)), s) + assert_series_equal(pd.Series(yaml.safe_load(y)), s) def test_frame_to_yaml_safe(): @@ -146,7 +146,7 @@ def test_frame_to_yaml_safe(): assert d == {'col1': {0: 100, 1: 200, 2: 300}, 'col2': {0: 'a', 1: 'b', 2: 'c'}} y = yaml.dump(d, default_flow_style=False) - assert_dfs_equal(pd.DataFrame(yaml.load(y)), df) + assert_dfs_equal(pd.DataFrame(yaml.safe_load(y)), df) def test_ordered_dict(): diff --git a/urbansim/utils/yamlio.py b/urbansim/utils/yamlio.py index 2e8391a6..476e29f8 100644 --- a/urbansim/utils/yamlio.py +++ b/urbansim/utils/yamlio.py @@ -220,7 +220,7 @@ def yaml_to_dict(yaml_str=None, str_or_buffer=None, ordered=False): if ordered: loader = __ordered_load else: - loader = yaml.load + loader = yaml.safe_load if yaml_str: d = loader(yaml_str) From 6546a407e190f7f15afd6ada5af0bd5406d9c420 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 10:38:43 -0700 Subject: [PATCH 14/46] Moving pandana and matplotlib to optional requirements --- requirements-dev.txt | 4 ++++ setup.py | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..4855110d --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +# Additional requirements for development and testing + +matplotlib +pandana >= 0.3 diff --git a/setup.py b/setup.py index c5c50bd9..7df256c9 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ packages=find_packages(exclude=['*.tests']), install_requires=[ 'bottle>=0.12', - 'matplotlib>=1.3.1', 'numpy>=1.8.0', 'orca>=1.1', 'pandas>=0.17.0', @@ -47,8 +46,5 @@ 'tables>=3.1.0', 'toolz>=0.7.0', 'zbox>=1.2' - ], - extras_require={ - 'pandana': ['pandana==0.3.0'] - } + ] ) From 2343daf263aea24516257bde66e404da11516206 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 10:53:45 -0700 Subject: [PATCH 15/46] Streamlining travis script --- .travis.yml | 60 ++++++++++++++++++-------------------------- requirements-dev.txt | 14 ++++++++++- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0486f6fd..6af605a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,44 +1,34 @@ language: python -sudo: false + python: -- '2.7' -- '3.6' -- '3.8' + - '2.7' + - '3.5' + - '3.6' + - '3.7' + - '3.8' + install: -- if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh - -O miniconda.sh; else wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - -O miniconda.sh; fi -- bash miniconda.sh -b -p $HOME/miniconda -- export PATH="$HOME/miniconda/bin:$PATH" -- hash -r -- conda config --set always_yes yes --set changeps1 no -- conda update -q conda -- conda info -a -- > - conda create -q -n test-environment - python=$TRAVIS_PYTHON_VERSION - cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy - statsmodels pytables pyyaml toolz -- source activate test-environment -- conda install pandana -c conda-forge -- pip install orca osmnet pandana bottle simplejson zbox prettytable -- pip install 'pytest<4.0' pytest-cov coveralls pycodestyle -- pip install . + - pip install . + - pip install -r requirements-dev.txt + - pip list + - pip show urbansim + # - conda: cytoolz jinja2 osmnet + before_script: -- git clone https://github.com/udst/sanfran_urbansim.git -- cd sanfran_urbansim; jupyter nbconvert --to python Simulation.ipynb -- cd .. + - git clone https://github.com/udst/sanfran_urbansim.git + - cd sanfran_urbansim; jupyter nbconvert --to python Simulation.ipynb + - cd .. + script: -- pycodestyle urbansim scripts -- py.test --cov urbansim --cov-report term-missing -- cd sanfran_urbansim; python Simulation.py -- cd .. + - pycodestyle urbansim scripts + - py.test --cov urbansim --cov-report term-missing + - cd sanfran_urbansim; python Simulation.py + - cd .. + after_success: -- coveralls -- bin/build_docs.sh -notifications: - slack: - secure: LnbM2nYkvZF7FreNhqG+ExMeIsbbOWuuOi8ewZCCvMEMsYBEc9oE1hi23enpGUMElGzCTKOdCpQ/Z/QNHMycGZaL/wAZm5EPGNM7CZG1FKDrPYOTJ18ipi+k72k3GuqXM3DwGzjcRO6sw7kdy1k/3uVu8QHmkdgijLT6smIh6r4= + - coveralls + - bin/build_docs.sh + env: global: - secure: c2olC+qzN1VHJZlznPy7mUtRvAyU7I2C7N3sYkv71Ds+dx5WN8InF6mpwNs/ZKfERbbA9slSVmYjB4j2c+VT8S1J33o6aI4F4hWNr6l7DgBKoPmozvaDx0GQ45hMOeSHSG5OF+DZCSTEXtxQK82uj61sCvZ6dIlDOqRvAHCjQDI= diff --git a/requirements-dev.txt b/requirements-dev.txt index 4855110d..d043c7dc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,16 @@ # Additional requirements for development and testing matplotlib -pandana >= 0.3 +pandana + +# testing +coveralls +jupyter +pycodestyle +pytest < 4.0 +pytest-cov + +# building documentation +numpydoc +sphinx +sphinx-rtd-theme From 0952c50d268cc98f78c06afe6e71be601461a02b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:03:13 -0700 Subject: [PATCH 16/46] Flagging more optional requirements --- .travis.yml | 2 +- requirements-dev.txt | 2 ++ setup.py | 24 +++++++++++------------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6af605a0..bddd6b6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,4 @@ after_success: env: global: - - secure: c2olC+qzN1VHJZlznPy7mUtRvAyU7I2C7N3sYkv71Ds+dx5WN8InF6mpwNs/ZKfERbbA9slSVmYjB4j2c+VT8S1J33o6aI4F4hWNr6l7DgBKoPmozvaDx0GQ45hMOeSHSG5OF+DZCSTEXtxQK82uj61sCvZ6dIlDOqRvAHCjQDI= + - secure: c2olC+qzN1VHJZlznPy7mUtRvAyU7I2C7N3sYkv71Ds+dx5WN8InF6mpwNs/ZKfERbbA9slSVmYjB4j2c+VT8S1J33o6aI4F4hWNr6l7DgBKoPmozvaDx0GQ45hMOeSHSG5OF+DZCSTEXtxQK82uj61sCvZ6dIlDOqRvAHCjQDI= diff --git a/requirements-dev.txt b/requirements-dev.txt index d043c7dc..635cd932 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,9 @@ # Additional requirements for development and testing +bottle matplotlib pandana +simplejson # testing coveralls diff --git a/setup.py b/setup.py index 7df256c9..aef1b4e4 100644 --- a/setup.py +++ b/setup.py @@ -33,18 +33,16 @@ }, packages=find_packages(exclude=['*.tests']), install_requires=[ - 'bottle>=0.12', - 'numpy>=1.8.0', - 'orca>=1.1', - 'pandas>=0.17.0', - 'patsy>=0.3.0', - 'prettytable>=0.7.2', - 'pyyaml>=3.10', - 'scipy>=0.13.3', - 'simplejson>=3.3', - 'statsmodels>=0.8.0', - 'tables>=3.1.0', - 'toolz>=0.7.0', - 'zbox>=1.2' + 'numpy >= 1.8.0', + 'orca >= 1.1', + 'pandas >= 0.17.0', + 'patsy >= 0.3.0', + 'prettytable >= 0.7.2', + 'pyyaml >= 3.10', + 'scipy >= 0.13.3', + 'statsmodels >= 0.8.0', + 'tables >= 3.1.0', + 'toolz >= 0.7.0', + 'zbox >= 1.2' ] ) From e4e04777736e359ec1da639b3ceabc7103815c4c Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:04:56 -0700 Subject: [PATCH 17/46] Fixing statsmodels version --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aef1b4e4..80ea41c4 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,8 @@ 'prettytable >= 0.7.2', 'pyyaml >= 3.10', 'scipy >= 0.13.3', - 'statsmodels >= 0.8.0', + 'statsmodels >= 0.8, <0.11; python_version <"3.6"', + 'statsmodels >= 0.8; python_version >="3.6"', 'tables >= 3.1.0', 'toolz >= 0.7.0', 'zbox >= 1.2' From 2ea6cbe09a760b0bdb96e28735663d3c813477a3 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:15:58 -0700 Subject: [PATCH 18/46] Updating toolz --- setup.py | 7 ++----- urbansim/accounts.py | 2 +- urbansim/models/dcm.py | 2 +- urbansim/models/regression.py | 2 +- urbansim/models/util.py | 2 +- urbansim/utils/misc.py | 2 +- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 80ea41c4..3f0c04f8 100644 --- a/setup.py +++ b/setup.py @@ -38,12 +38,9 @@ 'pandas >= 0.17.0', 'patsy >= 0.3.0', 'prettytable >= 0.7.2', - 'pyyaml >= 3.10', - 'scipy >= 0.13.3', + 'scipy >= 1.0', 'statsmodels >= 0.8, <0.11; python_version <"3.6"', 'statsmodels >= 0.8; python_version >="3.6"', - 'tables >= 3.1.0', - 'toolz >= 0.7.0', - 'zbox >= 1.2' + 'toolz >= 0.8.1' ] ) diff --git a/urbansim/accounts.py b/urbansim/accounts.py index a9c08061..e3ed7df6 100644 --- a/urbansim/accounts.py +++ b/urbansim/accounts.py @@ -5,7 +5,7 @@ from collections import namedtuple import pandas as pd -from zbox import toolz as tz +import toolz as tz Transaction = namedtuple('Transaction', ('amount', 'subaccount', 'metadata')) diff --git a/urbansim/models/dcm.py b/urbansim/models/dcm.py index fc62eeeb..f55c28d8 100644 --- a/urbansim/models/dcm.py +++ b/urbansim/models/dcm.py @@ -12,7 +12,7 @@ import pandas as pd from patsy import dmatrix from prettytable import PrettyTable -from zbox import toolz as tz +import toolz as tz from . import util from ..exceptions import ModelEvaluationError diff --git a/urbansim/models/regression.py b/urbansim/models/regression.py index 54298226..b80d363b 100644 --- a/urbansim/models/regression.py +++ b/urbansim/models/regression.py @@ -12,7 +12,7 @@ import statsmodels.formula.api as smf from patsy import dmatrix from prettytable import PrettyTable -from zbox import toolz as tz +import toolz as tz from . import util from ..exceptions import ModelEvaluationError diff --git a/urbansim/models/util.py b/urbansim/models/util.py index 799f1d94..868ea58b 100644 --- a/urbansim/models/util.py +++ b/urbansim/models/util.py @@ -14,7 +14,7 @@ import numpy as np import pandas as pd import patsy -from zbox import toolz as tz +import toolz as tz from ..utils.logutil import log_start_finish diff --git a/urbansim/utils/misc.py b/urbansim/utils/misc.py index e758a560..f5d410a6 100644 --- a/urbansim/utils/misc.py +++ b/urbansim/utils/misc.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from zbox import toolz as tz +import toolz as tz def _mkifnotexists(folder): From 33049af9771abb4846df031cf986ecfdf368236b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:21:10 -0700 Subject: [PATCH 19/46] Reinstating pyyaml --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f0c04f8..112eea52 100644 --- a/setup.py +++ b/setup.py @@ -22,10 +22,15 @@ license='BSD', url='https://github.com/udst/urbansim', classifiers=[ - 'Development Status :: 4 - Beta', + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'License :: OSI Approved :: BSD License' ], package_data={ @@ -38,6 +43,7 @@ 'pandas >= 0.17.0', 'patsy >= 0.3.0', 'prettytable >= 0.7.2', + 'pyyaml >= 3.10', 'scipy >= 1.0', 'statsmodels >= 0.8, <0.11; python_version <"3.6"', 'statsmodels >= 0.8; python_version >="3.6"', From d6b5d7852430701e5c8b910a9f776efbdba9925d Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:23:57 -0700 Subject: [PATCH 20/46] Dropping ez_setup --- ez_setup.py | 332 ---------------------------------------------------- setup.cfg | 2 +- setup.py | 8 -- 3 files changed, 1 insertion(+), 341 deletions(-) delete mode 100644 ez_setup.py diff --git a/ez_setup.py b/ez_setup.py deleted file mode 100644 index a34fa806..00000000 --- a/ez_setup.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python -"""Bootstrap setuptools installation - -To use setuptools in your package's setup.py, include this -file in the same directory and add this to the top of your setup.py:: - - from ez_setup import use_setuptools - use_setuptools() - -To require a specific version of setuptools, set a download -mirror, or use an alternate download directory, simply supply -the appropriate options to ``use_setuptools()``. - -This file can also be run as a script to install or upgrade setuptools. -""" -import os -import shutil -import sys -import tempfile -import zipfile -import optparse -import subprocess -import platform -import textwrap -import contextlib - -from distutils import log - -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - -try: - from site import USER_SITE -except ImportError: - USER_SITE = None - -DEFAULT_VERSION = "5.7" -DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" - -def _python_cmd(*args): - """ - Return True if the command succeeded. - """ - args = (sys.executable,) + args - return subprocess.call(args) == 0 - - -def _install(archive_filename, install_args=()): - with archive_context(archive_filename): - # installing - log.warn('Installing Setuptools') - if not _python_cmd('setup.py', 'install', *install_args): - log.warn('Something went wrong during the installation.') - log.warn('See the error message above.') - # exitcode will be 2 - return 2 - - -def _build_egg(egg, archive_filename, to_dir): - with archive_context(archive_filename): - # building an egg - log.warn('Building a Setuptools egg in %s', to_dir) - _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) - # returning the result - log.warn(egg) - if not os.path.exists(egg): - raise IOError('Could not build the egg.') - - -class ContextualZipFile(zipfile.ZipFile): - """ - Supplement ZipFile class to support context manager for Python 2.6 - """ - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __new__(cls, *args, **kwargs): - """ - Construct a ZipFile or ContextualZipFile as appropriate - """ - if hasattr(zipfile.ZipFile, '__exit__'): - return zipfile.ZipFile(*args, **kwargs) - return super(ContextualZipFile, cls).__new__(cls) - - -@contextlib.contextmanager -def archive_context(filename): - # extracting the archive - tmpdir = tempfile.mkdtemp() - log.warn('Extracting in %s', tmpdir) - old_wd = os.getcwd() - try: - os.chdir(tmpdir) - with ContextualZipFile(filename) as archive: - archive.extractall() - - # going in the directory - subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) - os.chdir(subdir) - log.warn('Now working in %s', subdir) - yield - - finally: - os.chdir(old_wd) - shutil.rmtree(tmpdir) - - -def _do_download(version, download_base, to_dir, download_delay): - egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' - % (version, sys.version_info[0], sys.version_info[1])) - if not os.path.exists(egg): - archive = download_setuptools(version, download_base, - to_dir, download_delay) - _build_egg(egg, archive, to_dir) - sys.path.insert(0, egg) - - # Remove previously-imported pkg_resources if present (see - # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). - if 'pkg_resources' in sys.modules: - del sys.modules['pkg_resources'] - - import setuptools - setuptools.bootstrap_install_from = egg - - -def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, download_delay=15): - to_dir = os.path.abspath(to_dir) - rep_modules = 'pkg_resources', 'setuptools' - imported = set(sys.modules).intersection(rep_modules) - try: - import pkg_resources - except ImportError: - return _do_download(version, download_base, to_dir, download_delay) - try: - pkg_resources.require("setuptools>=" + version) - return - except pkg_resources.DistributionNotFound: - return _do_download(version, download_base, to_dir, download_delay) - except pkg_resources.VersionConflict as VC_err: - if imported: - msg = textwrap.dedent(""" - The required version of setuptools (>={version}) is not available, - and can't be installed while this script is running. Please - install a more recent version first, using - 'easy_install -U setuptools'. - - (Currently using {VC_err.args[0]!r}) - """).format(VC_err=VC_err, version=version) - sys.stderr.write(msg) - sys.exit(2) - - # otherwise, reload ok - del pkg_resources, sys.modules['pkg_resources'] - return _do_download(version, download_base, to_dir, download_delay) - -def _clean_check(cmd, target): - """ - Run the command to download target. If the command fails, clean up before - re-raising the error. - """ - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if os.access(target, os.F_OK): - os.unlink(target) - raise - -def download_file_powershell(url, target): - """ - Download the file at url to target using Powershell (which will validate - trust). Raise an exception if the command cannot complete. - """ - target = os.path.abspath(target) - ps_cmd = ( - "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " - "[System.Net.CredentialCache]::DefaultCredentials; " - "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" - % vars() - ) - cmd = [ - 'powershell', - '-Command', - ps_cmd, - ] - _clean_check(cmd, target) - -def has_powershell(): - if platform.system() != 'Windows': - return False - cmd = ['powershell', '-Command', 'echo test'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_powershell.viable = has_powershell - -def download_file_curl(url, target): - cmd = ['curl', url, '--silent', '--output', target] - _clean_check(cmd, target) - -def has_curl(): - cmd = ['curl', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_curl.viable = has_curl - -def download_file_wget(url, target): - cmd = ['wget', url, '--quiet', '--output-document', target] - _clean_check(cmd, target) - -def has_wget(): - cmd = ['wget', '--version'] - with open(os.path.devnull, 'wb') as devnull: - try: - subprocess.check_call(cmd, stdout=devnull, stderr=devnull) - except Exception: - return False - return True - -download_file_wget.viable = has_wget - -def download_file_insecure(url, target): - """ - Use Python to download the file, even though it cannot authenticate the - connection. - """ - src = urlopen(url) - try: - # Read all the data in one block. - data = src.read() - finally: - src.close() - - # Write all the data in one block to avoid creating a partial file. - with open(target, "wb") as dst: - dst.write(data) - -download_file_insecure.viable = lambda: True - -def get_best_downloader(): - downloaders = ( - download_file_powershell, - download_file_curl, - download_file_wget, - download_file_insecure, - ) - viable_downloaders = (dl for dl in downloaders if dl.viable()) - return next(viable_downloaders, None) - -def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, - to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): - """ - Download setuptools from a specified location and return its filename - - `version` should be a valid setuptools version number that is available - as an sdist for download under the `download_base` URL (which should end - with a '/'). `to_dir` is the directory where the egg will be downloaded. - `delay` is the number of seconds to pause before an actual download - attempt. - - ``downloader_factory`` should be a function taking no arguments and - returning a function for downloading a URL to a target. - """ - # making sure we use the absolute path - to_dir = os.path.abspath(to_dir) - zip_name = "setuptools-%s.zip" % version - url = download_base + zip_name - saveto = os.path.join(to_dir, zip_name) - if not os.path.exists(saveto): # Avoid repeated downloads - log.warn("Downloading %s", url) - downloader = downloader_factory() - downloader(url, saveto) - return os.path.realpath(saveto) - -def _build_install_args(options): - """ - Build the arguments to 'python setup.py install' on the setuptools package - """ - return ['--user'] if options.user_install else [] - -def _parse_args(): - """ - Parse the command line for options - """ - parser = optparse.OptionParser() - parser.add_option( - '--user', dest='user_install', action='store_true', default=False, - help='install in user site package (requires Python 2.6 or later)') - parser.add_option( - '--download-base', dest='download_base', metavar="URL", - default=DEFAULT_URL, - help='alternative URL from where to download the setuptools package') - parser.add_option( - '--insecure', dest='downloader_factory', action='store_const', - const=lambda: download_file_insecure, default=get_best_downloader, - help='Use internal, non-validating downloader' - ) - parser.add_option( - '--version', help="Specify which version to download", - default=DEFAULT_VERSION, - ) - options, args = parser.parse_args() - # positional arguments are ignored - return options - -def main(): - """Install or upgrade setuptools and EasyInstall""" - options = _parse_args() - archive = download_setuptools( - version=options.version, - download_base=options.download_base, - downloader_factory=options.downloader_factory, - ) - return _install(archive, _build_install_args(options)) - -if __name__ == '__main__': - sys.exit(main()) diff --git a/setup.cfg b/setup.cfg index 6aeda162..14fa2f04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ -[pep8] +[pycodestyle] exclude = texttable.py max-line-length = 100 diff --git a/setup.py b/setup.py index 112eea52..56e7f789 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,5 @@ -# Install setuptools if not installed. -try: - import setuptools -except ImportError: - from ez_setup import use_setuptools - use_setuptools() - from setuptools import setup, find_packages - # read README as the long description with open('README.rst', 'r') as f: long_description = f.read() From cc60c799bb3514afd79dea66850d7b92ca0dc313 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:27:37 -0700 Subject: [PATCH 21/46] Cleanup --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bddd6b6b..b40a0d80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ install: - pip install -r requirements-dev.txt - pip list - pip show urbansim - # - conda: cytoolz jinja2 osmnet before_script: - git clone https://github.com/udst/sanfran_urbansim.git From 6d690e930fc88e578ee501c8a02ee9acfd6cb64d Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 11:34:57 -0700 Subject: [PATCH 22/46] Aligning appveyor with travis --- appveyor.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0ab7faf2..8abb5741 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,27 +4,29 @@ environment: matrix: - PYTHON_VERSION: 2.7 MINICONDA: C:\Miniconda + - PYTHON_VERSION: 3.5 + MINICONDA: C:\Miniconda3 - PYTHON_VERSION: 3.6 MINICONDA: C:\Miniconda3 + - PYTHON_VERSION: 3.7 + MINICONDA: C:\Miniconda3 + - PYTHON_VERSION: 3.8 + MINICONDA: C:\Miniconda3 init: - "ECHO %PYTHON_VERSION% %MINICONDA%" install: - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - "conda create -q -n test-environment python=%PYTHON_VERSION% cytoolz jupyter jinja2 matplotlib numpy pandas patsy pip scipy statsmodels pytables pyyaml toolz" - - activate test-environment - - conda install -c conda-forge shapely geopandas pytest=3.10 - - pip install orca pandana osmnet bottle simplejson zbox prettytable - - pip install pycodestyle + - conda update conda --yes --quiet + - "conda create --name test-env python=%PYTHON_VERSION% pip --yes --quiet" + - activate test-env - pip install . + - pip install -r requirements-dev.txt + - pip show urbansim before_test: -# Update sanfran_urbansim clone once python3 branch is merged - - git clone -b python3 https://github.com/udst/sanfran_urbansim.git + - git clone https://github.com/udst/sanfran_urbansim.git - cd sanfran_urbansim & jupyter nbconvert --to python Simulation.ipynb - cd.. From 6b2e28bb765e932bf106712d994f2e19305bfe4e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 12:11:21 -0700 Subject: [PATCH 23/46] Updating patsy requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 56e7f789..821bccd9 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'numpy >= 1.8.0', 'orca >= 1.1', 'pandas >= 0.17.0', - 'patsy >= 0.3.0', + 'patsy >= 0.4.1', 'prettytable >= 0.7.2', 'pyyaml >= 3.10', 'scipy >= 1.0', From 97365ea1b7523a04739b425228d4076986944b91 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 13:27:07 -0700 Subject: [PATCH 24/46] Adjusting appveyor environment --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 8abb5741..4a48a3ec 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,10 +16,13 @@ environment: init: - "ECHO %PYTHON_VERSION% %MINICONDA%" +# Requirements or sub-requirements that don't install well with pip +# are included directly in the conda environment + install: - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" - conda update conda --yes --quiet - - "conda create --name test-env python=%PYTHON_VERSION% pip --yes --quiet" + - "conda create --name test-env python=%PYTHON_VERSION% pip geopandas --yes --quiet" - activate test-env - pip install . - pip install -r requirements-dev.txt From 49657176c7c325eacbfd25264f5ffe7258942d01 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 13:48:22 -0700 Subject: [PATCH 25/46] Removing win python 3.8 --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 4a48a3ec..1819f802 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,8 +10,8 @@ environment: MINICONDA: C:\Miniconda3 - PYTHON_VERSION: 3.7 MINICONDA: C:\Miniconda3 - - PYTHON_VERSION: 3.8 - MINICONDA: C:\Miniconda3 + # - PYTHON_VERSION: 3.8 # doesn't run yet + # MINICONDA: C:\Miniconda3 init: - "ECHO %PYTHON_VERSION% %MINICONDA%" From 8f45f0f2f99986e3ff102a1296acc3998adb86d3 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Fri, 1 May 2020 16:31:44 -0700 Subject: [PATCH 26/46] Updating license --- LICENSE => LICENSE.txt | 2 +- MANIFEST.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename LICENSE => LICENSE.txt (96%) diff --git a/LICENSE b/LICENSE.txt similarity index 96% rename from LICENSE rename to LICENSE.txt index da52c471..c6b9d706 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2016, UrbanSim, Inc. All rights reserved. +Copyright (c) 2020, UrbanSim, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/MANIFEST.in b/MANIFEST.in index cafbd94a..97e2ad3d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include ez_setup.py +include LICENSE.txt include README.rst From 54e1c6e3a570d4084c1c46324cced19f632508c2 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 21:58:03 -0700 Subject: [PATCH 27/46] Draft changes to readme --- README.rst | 104 +++++++++-------------------------------------------- 1 file changed, 16 insertions(+), 88 deletions(-) diff --git a/README.rst b/README.rst index 975d1123..91da0275 100644 --- a/README.rst +++ b/README.rst @@ -1,105 +1,33 @@ UrbanSim ======== +.. image:: https://img.shields.io/pypi/v/urbansim.svg + :target: https://pypi.python.org/pypi/urbansim/ + :alt: Latest Version + .. image:: https://travis-ci.org/UDST/urbansim.svg?branch=master - :alt: Build Status (Linux) + :alt: Build Status :target: https://travis-ci.org/UDST/urbansim - -.. image:: https://ci.appveyor.com/api/projects/status/0ygo756020jpcrg3?svg=true - :alt: Build Status (Windows) - :target: https://ci.appveyor.com/project/pksohn/urbansim - - -.. image:: https://coveralls.io/repos/UDST/urbansim/badge.png?branch=master +.. image:: https://coveralls.io/repos/UDST/urbansim/badge.svg?branch=master :alt: Test Coverage :target: https://coveralls.io/r/UDST/urbansim?branch=master -New version of UrbanSim, a tool for modeling metropolitan real estate -markets - -.. image:: http://i.imgur.com/4YyN8ob.jpg - :alt: UrbanSim - -`Detailed documentation `__ for -UrbanSim is now available. - -`Click -here `__ -for installation instructions. - -Let us know what you are working on or if you think you have a great use case -by tweeting us at ``@urbansim`` or post on the UrbanSim `forum`_. - -UrbanSim History ----------------- - -UrbanSim (http://www.urbansim.com) is a model system for analyzing -urban development. It is an open source platform that has been -continuously refined and distributed for planning applications around -the world for over 15 years. Part of the evolution of the platform is -the necessary process of re-engineering the code to take advantage of -new developments in computational libraries and infrastructure. We -implemented UrbanSim initially in Java in the late 1990's, and by 2005 -determined that it was time to re-implement it in Python, and created -the Open Platform for Urban Simulation (OPUS) software implementation at -that time. Now, almost a decade later, it is time again to revisit the -implementation to take advantage of an amazing amount of innovation in -the scientific computing community. The new implementation is hosted on -this GitHub site, and maintained by UrbanSim Inc. and a growing -community of contributors. - -New UrbanSim Implementation ---------------------------- - -This new code base is a streamlined complete re-implementation of the -longstanding UrbanSim project aimed at *reducing the complexity* of -using the UrbanSim methodology. Redesigned from the ground up, the new -library is trivial to install, the development process is made -transparent via this GitHub site, and exhaustive documentation has been -created in the hopes of making modeling much more widely accessible to -planners and new modelers. - -We lean heavily on the `PyData `__ community to make -our work easier - Pandas, `IPython `__, and -`statsmodels `__ are ubiquitous in -this work. These Python libraries essentially replace the UrbanSim -Dataset class, tools to read and write from other storage, and some of -the statistical estimation previously implemented by UrbanSim. - -This makes our task easier as we can focus on urban modeling and leave -the infrastructure to the wider Python community. The -`Pandas `__ library is the core of the new -UrbanSim, which is an extremely popular data manipulation library with a -large community providing support and a very helpful -`book `__. - -We have now converted a full set of UrbanSim models to the new -framework, and have running applications for the Paris, Albuquerque, -Denver, Bay Area, and Detroit regions. We have implemented a complete -set of hedonic price models, location choice models, relocation and -transition models, as well as a new real estate development model using -proforma analysis. +UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. -We do strongly recommend that you contact the team at www.urbansim.com about your -project to make sure you can get support when you need it, -and know what you are getting into. For major applied projects, -professional support is highly recommended. +This UrbanSim *Python library* is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. -Reporting bugs and contributing to UrbanSim --------------------------------------------- +Operational UrbanSim models begin with detailed data about a particular region, and then estimate and validate a system of interconnected model components. Full models draw on a number of libraries: not just ``urbansim``, but also Orca for task orchestration, Synthpop for population synthesis, Pandana for network analysis, Developer for real estate logic, and so on. Collectively, these make up the `Urban Data Science Toolkit `__ (UDST). -Please report any bugs you encounter via `GitHub Issues `__. +UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by UrbanSim Inc. -If you have improvements or new features you would like to see in UrbanSim: +Learn More +---------- -1. Open a feature request via `GitHub Issues `__. -2. See our code contribution instructions `here `__. -3. Contribute your code from a fork or branch by using a Pull Request and request a review so it can be considered as an addition to the codebase. +* `An Introduction to UrbanSim `__ -Academic literature -------------------- +* `Documentation `__ for the ``urbansim`` Python library -A selection of academic literature on UrbanSim can be found `here `_. +* `UrbanSim for San Francisco: An example implementation `__ -.. _forum: http://discussion.urbansim.com/ \ No newline at end of file +* `UrbanSim Inc. `__ From 712bbb99c4a0efabeb4d01ab23dcc5adc8558935 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 22:10:37 -0700 Subject: [PATCH 28/46] Updates to readme --- README.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 91da0275..1910df59 100644 --- a/README.rst +++ b/README.rst @@ -15,9 +15,12 @@ UrbanSim UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. -This UrbanSim *Python library* is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. +This ``urbansim``` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. -Operational UrbanSim models begin with detailed data about a particular region, and then estimate and validate a system of interconnected model components. Full models draw on a number of libraries: not just ``urbansim``, but also Orca for task orchestration, Synthpop for population synthesis, Pandana for network analysis, Developer for real estate logic, and so on. Collectively, these make up the `Urban Data Science Toolkit `__ (UDST). +How it works +------------ + +Operational UrbanSim models begin with detailed data about a particular region, and then estimate and validate a system of interconnected model components. Full models draw on a number of libraries: not just ``urbansim``, but also `Orca `__ for task orchestration, `Synthpop `__ for population synthesis, `Pandana `__ for network analysis, and so on. Collectively, these make up the `Urban Data Science Toolkit `__ (UDST). UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by UrbanSim Inc. @@ -26,8 +29,15 @@ Learn More * `An Introduction to UrbanSim `__ -* `Documentation `__ for the ``urbansim`` Python library - * `UrbanSim for San Francisco: An example implementation `__ * `UrbanSim Inc. `__ + +Technical documentation +----------------------- + +* `Documentation `__ for the ``urbansim`` library + +* `Urban Data Science Toolkit `__ + +* Documentation for the `UrbanSim Cloud Platform `__ From 7347d37f31c016786005a68074b5dfce13547e5c Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 22:13:54 -0700 Subject: [PATCH 29/46] Fixing a typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1910df59..49d63cfc 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ UrbanSim UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. -This ``urbansim``` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. +This ``urbansim`` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. How it works ------------ From 3dbf2974968bed503d8f3f86c75016431475becb Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 22:40:17 -0700 Subject: [PATCH 30/46] Adding installation instructions --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 49d63cfc..d16edf34 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,13 @@ Learn More * `UrbanSim Inc. `__ +Installation +------------ + +* ``pip install urbansim`` + +* ``conda install urbansim --channel conda-forge`` + Technical documentation ----------------------- From 12fdb3201a7a784346adfa0a0f5fbb59fa7d5610 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 22:47:39 -0700 Subject: [PATCH 31/46] Reformatting --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d16edf34..da8f91b1 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ Operational UrbanSim models begin with detailed data about a particular region, UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by UrbanSim Inc. Learn More ----------- +~~~~~~~~~~ * `An Introduction to UrbanSim `__ From fe07b22506cca04443dab5bb30740878b4579478 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 4 May 2020 22:49:05 -0700 Subject: [PATCH 32/46] Reformatting --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index da8f91b1..d16edf34 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ Operational UrbanSim models begin with detailed data about a particular region, UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by UrbanSim Inc. Learn More -~~~~~~~~~~ +---------- * `An Introduction to UrbanSim `__ From 4d562355a3af32e92ba5543864785ced33d0e2be Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 09:45:17 -0700 Subject: [PATCH 33/46] Removing development scripts --- scripts/cache_to_hdf5.py | 155 --------------------------------------- scripts/view_hdf5.py | 18 ----- 2 files changed, 173 deletions(-) delete mode 100755 scripts/cache_to_hdf5.py delete mode 100644 scripts/view_hdf5.py diff --git a/scripts/cache_to_hdf5.py b/scripts/cache_to_hdf5.py deleted file mode 100755 index d9cdece9..00000000 --- a/scripts/cache_to_hdf5.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import argparse -import glob -import os -import sys - -import numpy as np -import pandas as pd - - -def cache_to_df(dir_path): - """ - Convert a directory of binary array data files to a Pandas DataFrame. - - Parameters - ---------- - dir_path : str - - - """ - table = {} - for attrib in glob.glob(os.path.join(dir_path, '*')): - attrib_name, attrib_ext = os.path.splitext(os.path.basename(attrib)) - if attrib_ext == '.lf8': - attrib_data = np.fromfile(attrib, np.float64) - table[attrib_name] = attrib_data - - elif attrib_ext == '.lf4': - attrib_data = np.fromfile(attrib, np.float32) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li2': - attrib_data = np.fromfile(attrib, np.int16) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li4': - attrib_data = np.fromfile(attrib, np.int32) - table[attrib_name] = attrib_data - - elif attrib_ext == '.li8': - attrib_data = np.fromfile(attrib, np.int64) - table[attrib_name] = attrib_data - - elif attrib_ext == '.ib1': - attrib_data = np.fromfile(attrib, np.bool_) - table[attrib_name] = attrib_data - - elif attrib_ext.startswith('.iS'): - length_string = int(attrib_ext[3:]) - attrib_data = np.fromfile(attrib, ('a' + str(length_string))) - table[attrib_name] = attrib_data - - else: - print('Array {} is not a recognized data type'.format(attrib)) - - df = pd.DataFrame(table) - return df - - -DIRECTORIES = { - 'parcels', 'buildings', 'households', 'jobs', 'zones', 'travel_data', - 'annual_employment_control_totals', 'annual_household_control_totals', - 'annual_household_relocation_rates', 'annual_job_relocation_rates', - 'building_sqft_per_job', 'building_types', 'counties', 'target_vacancies', - 'development_event_history' -} - - -def convert_dirs(base_dir, hdf_name, complib=None, complevel=0): - """ - Convert nested set of directories to - - """ - print('Converting directories in {}'.format(base_dir)) - - dirs = glob.glob(os.path.join(base_dir, '*')) - dirs = {d for d in dirs if os.path.basename(d) in DIRECTORIES} - if not dirs: - raise RuntimeError('No direcotries found matching known data.') - - store = pd.HDFStore( - hdf_name, mode='w', complevel=complevel, complib=complib) - - for dirpath in dirs: - dirname = os.path.basename(dirpath) - - print(dirname) - df = cache_to_df(dirpath) - - if dirname == 'travel_data': - keys = ['from_zone_id', 'to_zone_id'] - elif dirname == 'annual_employment_control_totals': - keys = ['sector_id', 'year', 'home_based_status'] - elif dirname == 'annual_job_relocation_rates': - keys = ['sector_id'] - elif dirname == 'annual_household_control_totals': - keys = ['year'] - elif dirname == 'annual_household_relocation_rates': - keys = ['age_of_head_max', 'age_of_head_min', - 'income_min', 'income_max'] - elif dirname == 'building_sqft_per_job': - keys = ['zone_id', 'building_type_id'] - elif dirname == 'counties': - keys = ['county_id'] - elif dirname == 'development_event_history': - keys = ['building_id'] - elif dirname == 'target_vacancies': - keys = ['building_type_id', 'year'] - else: - keys = [dirname[:-1] + '_id'] - - if dirname != 'annual_household_relocation_rates': - df = df.set_index(keys) - - for colname in df.columns: - if df[colname].dtype == np.float64: - df[colname] = df[colname].astype(np.float32) - elif df[colname].dtype == np.int64: - df[colname] = df[colname].astype(np.int32) - else: - df[colname] = df[colname] - - df.info() - print(os.linesep) - store.put(dirname, df) - - store.close() - - -def parse_args(args=None): - parser = argparse.ArgumentParser( - description=( - 'Convert nested set of directories containing binary ' - 'array data to an HDF5 file made from Pandas DataFrames.')) - parser.add_argument('base_dir', help='Base data directory for conversion.') - parser.add_argument('hdf_name', help='Name of output HDF5 file.') - parser.add_argument('-c', '--complib', - help=('Compression library to use, if any. ' - "Can be one of " - "'zlib', 'bzip2', 'lzo', 'blosc'.")) - parser.add_argument('-l', '--complevel', type=int, default=0, - help='Compression level to use.') - return parser.parse_args(args) - - -def main(args=None): - args = parse_args(args) - convert_dirs(args.base_dir, args.hdf_name, args.complib, args.complevel) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/scripts/view_hdf5.py b/scripts/view_hdf5.py deleted file mode 100644 index 47e9f0f4..00000000 --- a/scripts/view_hdf5.py +++ /dev/null @@ -1,18 +0,0 @@ -import json -import sys - -import pandas as pd - -args = sys.argv[1:] - -assert len(args) == 1 - -store = pd.HDFStore(args[0], "r") - -print store - -for key in store.keys(): - print "\n\nTABLENAME", key - print store[key] - print store[key].dtypes - print store[key].describe() From 81e985709b9cfb2b60f6b8fa563d17b0972173f1 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 09:50:14 -0700 Subject: [PATCH 34/46] Removing scripts from CI tests --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b40a0d80..06a06609 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_script: - cd .. script: - - pycodestyle urbansim scripts + - pycodestyle urbansim - py.test --cov urbansim --cov-report term-missing - cd sanfran_urbansim; python Simulation.py - cd .. diff --git a/appveyor.yml b/appveyor.yml index 1819f802..57e6c36b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,6 +34,6 @@ before_test: - cd.. test_script: - - pycodestyle urbansim scripts + - pycodestyle urbansim - pytest urbansim - cd sanfran_urbansim & python Simulation.py \ No newline at end of file From a765416320b03f711bbbedfc82c75c739311f1fd Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 09:55:00 -0700 Subject: [PATCH 35/46] Removing special coverage settings --- .coveragerc | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index d3d73d75..00000000 --- a/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - urbansim/urbanchoice/pmat.py - urbansim/**/tests/* From ffb677a18c1be572f1163acb59f81f26ad6c60e9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 09:59:52 -0700 Subject: [PATCH 36/46] Removing special pycodestyle settings --- .travis.yml | 2 +- appveyor.yml | 2 +- setup.cfg | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 setup.cfg diff --git a/.travis.yml b/.travis.yml index 06a06609..cca4d5e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ before_script: - cd .. script: - - pycodestyle urbansim + - pycodestyle --max-line-length=100 urbansim - py.test --cov urbansim --cov-report term-missing - cd sanfran_urbansim; python Simulation.py - cd .. diff --git a/appveyor.yml b/appveyor.yml index 57e6c36b..174546d5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,6 +34,6 @@ before_test: - cd.. test_script: - - pycodestyle urbansim + - pycodestyle --max-line-length=100 urbansim - pytest urbansim - cd sanfran_urbansim & python Simulation.py \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 14fa2f04..00000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[pycodestyle] -exclude = texttable.py -max-line-length = 100 From 1f188043170e155f8abb9620c876ff81d0bcaa2b Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 10:08:11 -0700 Subject: [PATCH 37/46] Renaming HISTORY -> CHANGELOG --- HISTORY.rst => CHANGELOG.rst | 0 MANIFEST.in | 2 ++ 2 files changed, 2 insertions(+) rename HISTORY.rst => CHANGELOG.rst (100%) diff --git a/HISTORY.rst b/CHANGELOG.rst similarity index 100% rename from HISTORY.rst rename to CHANGELOG.rst diff --git a/MANIFEST.in b/MANIFEST.in index 97e2ad3d..ef85ef0c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ +include CHANGELOG.rst include LICENSE.txt include README.rst +include requirements-dev.txt \ No newline at end of file From 92a72507ca123fad759be43628a651bca86b7fc0 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 13:40:53 -0700 Subject: [PATCH 38/46] Updating and documenting doc build process --- .travis.yml | 5 -- bin/build_docs.sh | 61 ------------------ docs/README.md | 24 +++++++ docs/build/.gitignore | 1 + docs/{ => source}/Makefile | 0 docs/{ => source}/_static/.placeholder | 0 docs/{ => source}/_templates/.placeholder | 0 docs/{ => source}/conf.py | 0 docs/{ => source}/developer/index.rst | 0 docs/{ => source}/examples.rst | 0 docs/{ => source}/gettingstarted.rst | 0 docs/{ => source}/index.rst | 0 docs/{ => source}/make.bat | 0 docs/{ => source}/maps/index.rst | 0 docs/{ => source}/models/index.rst | 0 docs/{ => source}/models/statistical.rst | 0 docs/{ => source}/models/supplydemand.rst | 0 docs/{ => source}/models/transrelo.rst | 0 docs/{ => source}/models/urbanchoice.rst | 0 docs/{ => source}/models/util.rst | 0 .../dframe_explorer_screenshot.png | Bin docs/{ => source}/utils/index.rst | 0 docs/{ => source}/utils/misc.rst | 0 docs/{ => source}/utils/testing.rst | 0 docs/{ => source}/utils/yamlio.rst | 0 25 files changed, 25 insertions(+), 66 deletions(-) delete mode 100755 bin/build_docs.sh create mode 100644 docs/README.md create mode 100644 docs/build/.gitignore rename docs/{ => source}/Makefile (100%) rename docs/{ => source}/_static/.placeholder (100%) rename docs/{ => source}/_templates/.placeholder (100%) rename docs/{ => source}/conf.py (100%) rename docs/{ => source}/developer/index.rst (100%) rename docs/{ => source}/examples.rst (100%) rename docs/{ => source}/gettingstarted.rst (100%) rename docs/{ => source}/index.rst (100%) rename docs/{ => source}/make.bat (100%) rename docs/{ => source}/maps/index.rst (100%) rename docs/{ => source}/models/index.rst (100%) rename docs/{ => source}/models/statistical.rst (100%) rename docs/{ => source}/models/supplydemand.rst (100%) rename docs/{ => source}/models/transrelo.rst (100%) rename docs/{ => source}/models/urbanchoice.rst (100%) rename docs/{ => source}/models/util.rst (100%) rename docs/{ => source}/screenshots/dframe_explorer_screenshot.png (100%) rename docs/{ => source}/utils/index.rst (100%) rename docs/{ => source}/utils/misc.rst (100%) rename docs/{ => source}/utils/testing.rst (100%) rename docs/{ => source}/utils/yamlio.rst (100%) diff --git a/.travis.yml b/.travis.yml index cca4d5e3..e94498cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,3 @@ script: after_success: - coveralls - - bin/build_docs.sh - -env: - global: - - secure: c2olC+qzN1VHJZlznPy7mUtRvAyU7I2C7N3sYkv71Ds+dx5WN8InF6mpwNs/ZKfERbbA9slSVmYjB4j2c+VT8S1J33o6aI4F4hWNr6l7DgBKoPmozvaDx0GQ45hMOeSHSG5OF+DZCSTEXtxQK82uj61sCvZ6dIlDOqRvAHCjQDI= diff --git a/bin/build_docs.sh b/bin/build_docs.sh deleted file mode 100755 index 9360bb40..00000000 --- a/bin/build_docs.sh +++ /dev/null @@ -1,61 +0,0 @@ -#! /usr/bin/env bash - -# Copied from github.com/sympy/sympy -# -# This file automatically deploys changes to http://udst.github.io/urbansim/. -# This will only happen when building a non-pull request build on the master -# branch of UrbanSim. -# It requires an access token which should be present in .travis.yml file. -# -# Following is the procedure to get the access token: -# -# $ curl -X POST -u -H "Content-Type: application/json" -d\ -# "{\"scopes\":[\"public_repo\"],\"note\":\"token for pushing from travis\"}"\ -# https://api.github.com/authorizations -# -# It'll give you a JSON response having a key called "token". -# -# $ gem install travis -# $ travis encrypt -r sympy/sympy GH_TOKEN= env.global -# -# This will give you an access token("secure"). This helps in creating an -# environment variable named GH_TOKEN while building. -# -# Add this secure code to .travis.yml as described here http://docs.travis-ci.com/user/encryption-keys/ - -# Exit on error -set -e - -ACTUAL_TRAVIS_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER| cut -d'.' -f 2` - -if [ "$TRAVIS_REPO_SLUG" == "UDST/urbansim" ] && \ - [ "$TRAVIS_BRANCH" == "master" ] && \ - [ "$TRAVIS_PULL_REQUEST" == "false" ] && \ - [ "$ACTUAL_TRAVIS_JOB_NUMBER" == "1" ]; then - - echo "Installing dependencies" - conda install --yes --quiet sphinx numpydoc - pip install sphinx_rtd_theme - - echo "Building docs" - cd docs - make clean - make html - - cd ../../ - echo "Setting git attributes" - git config --global user.email "jiffyclub@gmail.com" - git config --global user.name "Matt Davis" - - echo "Cloning repository" - git clone --quiet --single-branch --branch=gh-pages https://${GH_TOKEN}@github.com/UDST/urbansim.git gh-pages > /dev/null 2>&1 - - cd gh-pages - rm -rf * - cp -R ../urbansim/docs/_build/html/* ./ - git add -A . - - git commit -am "Update dev doc after building $TRAVIS_BUILD_NUMBER" - echo "Pushing commit" - git push -fq origin gh-pages > /dev/null 2>&1 -fi diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..714b86c8 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,24 @@ +This folder generates the UrbanSim online documentation, hosted at https://udst.github.io/urbansim/. + +### How it works + +Documentation is generated using [Sphinx](http://sphinx-doc.org) and hosted with Github Pages from the `gh-pages` branch of the repository. The online documentation is updated **manually**. + +### Editing the documentation + +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for the Sphinx documentation generator. + +### Previewing changes locally + +Install the copy of UrbanSim that the documentation is meant to reflect. Install the documentation tools. + +``` +pip install . +pip install sphinx sphinx_rtd_theme numpydoc +``` + +From the `docs` directory, run `sphinx-build -b html source build`. The HTML files will show up in `docs/build/`. + +### Uploading changes + +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to Github. diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 00000000..150f68c8 --- /dev/null +++ b/docs/build/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/docs/Makefile b/docs/source/Makefile similarity index 100% rename from docs/Makefile rename to docs/source/Makefile diff --git a/docs/_static/.placeholder b/docs/source/_static/.placeholder similarity index 100% rename from docs/_static/.placeholder rename to docs/source/_static/.placeholder diff --git a/docs/_templates/.placeholder b/docs/source/_templates/.placeholder similarity index 100% rename from docs/_templates/.placeholder rename to docs/source/_templates/.placeholder diff --git a/docs/conf.py b/docs/source/conf.py similarity index 100% rename from docs/conf.py rename to docs/source/conf.py diff --git a/docs/developer/index.rst b/docs/source/developer/index.rst similarity index 100% rename from docs/developer/index.rst rename to docs/source/developer/index.rst diff --git a/docs/examples.rst b/docs/source/examples.rst similarity index 100% rename from docs/examples.rst rename to docs/source/examples.rst diff --git a/docs/gettingstarted.rst b/docs/source/gettingstarted.rst similarity index 100% rename from docs/gettingstarted.rst rename to docs/source/gettingstarted.rst diff --git a/docs/index.rst b/docs/source/index.rst similarity index 100% rename from docs/index.rst rename to docs/source/index.rst diff --git a/docs/make.bat b/docs/source/make.bat similarity index 100% rename from docs/make.bat rename to docs/source/make.bat diff --git a/docs/maps/index.rst b/docs/source/maps/index.rst similarity index 100% rename from docs/maps/index.rst rename to docs/source/maps/index.rst diff --git a/docs/models/index.rst b/docs/source/models/index.rst similarity index 100% rename from docs/models/index.rst rename to docs/source/models/index.rst diff --git a/docs/models/statistical.rst b/docs/source/models/statistical.rst similarity index 100% rename from docs/models/statistical.rst rename to docs/source/models/statistical.rst diff --git a/docs/models/supplydemand.rst b/docs/source/models/supplydemand.rst similarity index 100% rename from docs/models/supplydemand.rst rename to docs/source/models/supplydemand.rst diff --git a/docs/models/transrelo.rst b/docs/source/models/transrelo.rst similarity index 100% rename from docs/models/transrelo.rst rename to docs/source/models/transrelo.rst diff --git a/docs/models/urbanchoice.rst b/docs/source/models/urbanchoice.rst similarity index 100% rename from docs/models/urbanchoice.rst rename to docs/source/models/urbanchoice.rst diff --git a/docs/models/util.rst b/docs/source/models/util.rst similarity index 100% rename from docs/models/util.rst rename to docs/source/models/util.rst diff --git a/docs/screenshots/dframe_explorer_screenshot.png b/docs/source/screenshots/dframe_explorer_screenshot.png similarity index 100% rename from docs/screenshots/dframe_explorer_screenshot.png rename to docs/source/screenshots/dframe_explorer_screenshot.png diff --git a/docs/utils/index.rst b/docs/source/utils/index.rst similarity index 100% rename from docs/utils/index.rst rename to docs/source/utils/index.rst diff --git a/docs/utils/misc.rst b/docs/source/utils/misc.rst similarity index 100% rename from docs/utils/misc.rst rename to docs/source/utils/misc.rst diff --git a/docs/utils/testing.rst b/docs/source/utils/testing.rst similarity index 100% rename from docs/utils/testing.rst rename to docs/source/utils/testing.rst diff --git a/docs/utils/yamlio.rst b/docs/source/utils/yamlio.rst similarity index 100% rename from docs/utils/yamlio.rst rename to docs/source/utils/yamlio.rst From 59f4635e452bb1cbcc06dde4b214110397dc82be Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 14:52:12 -0700 Subject: [PATCH 39/46] Updating documentation --- docs/README.md | 13 +++- docs/source/conf.py | 6 +- docs/source/gettingstarted.rst | 117 ++++++--------------------------- docs/source/index.rst | 9 +-- setup.py | 2 +- 5 files changed, 40 insertions(+), 107 deletions(-) diff --git a/docs/README.md b/docs/README.md index 714b86c8..1021b80b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,11 +2,11 @@ This folder generates the UrbanSim online documentation, hosted at https://udst. ### How it works -Documentation is generated using [Sphinx](http://sphinx-doc.org) and hosted with Github Pages from the `gh-pages` branch of the repository. The online documentation is updated **manually**. +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with Github Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. ### Editing the documentation -The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for the Sphinx documentation generator. +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for Sphinx. ### Previewing changes locally @@ -17,7 +17,14 @@ pip install . pip install sphinx sphinx_rtd_theme numpydoc ``` -From the `docs` directory, run `sphinx-build -b html source build`. The HTML files will show up in `docs/build/`. +Build the documentation. There should be status messages and warnings, but no errors. + +``` +cd docs +sphinx-build -b html source build +``` + +The HTML files will show up in `docs/build/`. ### Uploading changes diff --git a/docs/source/conf.py b/docs/source/conf.py index 6d442eef..ef1a8656 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -50,16 +50,16 @@ # General information about the project. project = u'urbansim' -copyright = u'2017, UrbanSim Inc' +copyright = u'2020, UrbanSim Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.1' +version = '3.2' # The full version, including alpha/beta/rc tags. -release = '3.1.1' +release = '3.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/gettingstarted.rst b/docs/source/gettingstarted.rst index 0e7ad269..1c368a89 100644 --- a/docs/source/gettingstarted.rst +++ b/docs/source/gettingstarted.rst @@ -2,119 +2,47 @@ Getting Started =============== Let us know what you are working on or if you think you have a great use case -by tweeting us at ``@urbansim`` or post on the UrbanSim `forum`_. +by tweeting us at `@urbansim `__, posting on the UrbanSim `forum `__, or contacting us at info@urbansim.com. Installation ------------ -.. note:: - In the instructions below we will direct you to run various commands. - On Mac and Linux these should go in your standard terminal. - On Windows you may use the standard command prompt, the Anaconda - command prompt, or even Git Bash (if you have that installed). +The UrbanSim library is currently tested with Python versions 2.7, 3.5, 3.6, 3.7, and 3.8. -Anaconda -~~~~~~~~ +UrbanSim is distributed on the `Python Package Index `__ (for pip) and on `Conda Forge `__. The official source code is hosted on `GitHub `__. -UrbanSim is a Python library that uses a number of packages from the -scientific Python ecosystem. -The easiest way to get your own scientific Python installation is to -install `Anaconda `__, -which contains most of the libraries upon which UrbanSim depends. +You can install UrbanSim with either the pip or conda package manager: -UrbanSim -~~~~~~~~ +.. code-block:: python -Dependencies -^^^^^^^^^^^^ - -UrbanSim depends on the following libraries, most of which are in Anaconda: - -* `bottle `__ >= 0.12 -* `matplotlib `__ >= 1.3.1 -* `numpy `__ >= 1.8.0 -* `orca `__ >= 1.1 -* `pandas `__ >= 0.17.0 -* `patsy `__ >= 0.3.0 -* `prettytable `__ >= 0.7.2 -* `pyyaml `__ >= 3.10 -* `scipy `__ >= 0.13.3 -* `simplejson `__ >= 3.3 -* `statsmodels `__ >= 0.8.0 -* `tables `__ >= 3.1.0 -* `toolz `__ >= 0.7 -* `zbox `__ >= 1.2 - -Extras require: - -* `pandana `__ >= 0.1 - -Latest Release -^^^^^^^^^^^^^^ - -conda -##### - -`conda `__, which comes with Anaconda, is the -easiest way to install UrbanSim because it has binary installers for -all of UrbanSim's hard-to-install dependencies. -First, add the `udst channel `__ -to your conda configuration:: - - conda config --add channels udst - -Then use conda to install UrbanSim:: - - conda install urbansim - -To update to a new UrbanSim version use the ``conda update`` command:: - - conda update urbansim - -pip -### + pip install urbansim -UrbanSim can also be installed from -`PyPI `__ -via `pip `__:: +.. code-block:: python - pip install urbansim + conda install urbansim --channel conda-forge -When using this method it's best to already have most of the dependencies -installed, otherwise pip will try to download and install things like -NumPy, SciPy, and matplotlib. -If you're using Anaconda you will already have all of the hard-to-install -libraries. +Dependencies include `NumPy `__, `Pandas `__, and `Statsmodels `__, plus another UDST library: `Orca `__ for task orchestration. These will be installed automatically if needed. -To update to a new release of UrbanSim use the ``-U`` option with -``pip install``:: +When new releases of UrbanSim come out, you can upgrade like this: - pip install -U urbansim +.. code-block:: python -Development Version -^^^^^^^^^^^^^^^^^^^ + pip install urbansim --upgrade -UrbanSim can be installed from our -`development repository `__ -using `pip `__, a Python package manager. -pip is included with Anaconda so you should now be able to open a terminal -and run the following command to install UrbanSim:: +.. code-block:: python - pip install -U https://github.com/udst/urbansim/archive/master.zip + conda update urbansim --channel conda-forge -This will download urbansim and install the remaining dependencies not -included in Anaconda. +Installing from GitHub +~~~~~~~~~~~~~~~~~~~~~~ -If you need to update UrbanSim run the above command again. +You can also install UrbanSim directly from source code on GitHub, for example to use pre-release features or to modify the code yourself. The UrbanSim library is written entirely in Python; no compilation is needed. -Developer Install -^^^^^^^^^^^^^^^^^ +.. code-block:: python -If you are going to be developing on UrbanSim you will want to fork our -`GitHub repository `_ and clone -your fork to your computer. Then run ``python setup.py develop`` to install -UrbanSim in developer mode. In this mode you won't have to reinstall -UrbanSim every time you make changes. + git clone -b branch-name https://github.com/udst/urbansim.git + cd urbansim + pip install . Reporting bugs and contributing to UrbanSim ------------------------------------------- @@ -429,6 +357,3 @@ can be added to the same file so that a piece of functionality can be separated be kept together, columns together, and tables together - the organization is up to you. We hope that this flexibility inspires innovation for specific use cases, but what follows is a set of tutorials that we consider best practices. - - -.. _forum: http://discussion.urbansim.com/ \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 986d03ef..49663e49 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,10 +6,11 @@ UrbanSim ======== -`UrbanSim `_ is a model system for analyzing urban -development. -It is an open source platform that has been continuously refined and -distributed for planning applications around the world for over 15 years. +UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. + +This ``urbansim``` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. + +v3.2, released May 5, 2020 Contents -------- diff --git a/setup.py b/setup.py index 821bccd9..b168f304 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='urbansim', version='3.1.1', - description='Tool for modeling metropolitan real estate markets', + description='Platform for building statistical models of cities and regions', long_description=long_description, author='UrbanSim Inc.', author_email='info@urbansim.com', From 9313d119a6ee8e34a533e35324372440f286312a Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 14:52:37 -0700 Subject: [PATCH 40/46] Updating version number --- setup.py | 2 +- urbansim/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b168f304..fa4b5c74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='urbansim', - version='3.1.1', + version='3.2', description='Platform for building statistical models of cities and regions', long_description=long_description, author='UrbanSim Inc.', diff --git a/urbansim/__init__.py b/urbansim/__init__.py index abd22615..59cbdc33 100644 --- a/urbansim/__init__.py +++ b/urbansim/__init__.py @@ -1 +1 @@ -__version__ = version = '3.1.1' +__version__ = version = '3.2' From 04c8f53fb3a285421a7b7e5456277e80e7053233 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 15:03:39 -0700 Subject: [PATCH 41/46] Changelog and docs --- CHANGELOG.rst | 10 ++++++++++ docs/README.md | 4 ++-- docs/source/gettingstarted.rst | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c374db45..5935ff95 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,13 @@ +v3.2 +==== + +2020/05/05 + +* Improved installation and compatibility +* Support for Pandas 1.0 +* Various improvements and bug fixes +* Note that active development of certain UrbanSim components has moved to stand-alone libraries in UDST: Developer, Choicemodels, UrbanSim Templates + v3.1.1 ====== diff --git a/docs/README.md b/docs/README.md index 1021b80b..3f4a308e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ This folder generates the UrbanSim online documentation, hosted at https://udst. ### How it works -HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with Github Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with GitHub Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. ### Editing the documentation @@ -28,4 +28,4 @@ The HTML files will show up in `docs/build/`. ### Uploading changes -Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to Github. +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to GitHub. diff --git a/docs/source/gettingstarted.rst b/docs/source/gettingstarted.rst index 1c368a89..770b65cd 100644 --- a/docs/source/gettingstarted.rst +++ b/docs/source/gettingstarted.rst @@ -1,7 +1,7 @@ Getting Started =============== -Let us know what you are working on or if you think you have a great use case +Let us know what you are working on, or if you think you have a great use case, by tweeting us at `@urbansim `__, posting on the UrbanSim `forum `__, or contacting us at info@urbansim.com. Installation @@ -9,9 +9,9 @@ Installation The UrbanSim library is currently tested with Python versions 2.7, 3.5, 3.6, 3.7, and 3.8. -UrbanSim is distributed on the `Python Package Index `__ (for pip) and on `Conda Forge `__. The official source code is hosted on `GitHub `__. +UrbanSim is distributed on the `Python Package Index `__ (for Pip) and on `Conda Forge `__. The official source code is hosted on `GitHub `__. (UrbanSim versions before 3.2 are on the UDST Conda channel rather than Conda Forge.) -You can install UrbanSim with either the pip or conda package manager: +You can install UrbanSim with either the Pip or Conda package manager: .. code-block:: python From be859096690d9b15c494efe3b08efa1d8bc1a908 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 15:17:31 -0700 Subject: [PATCH 42/46] Updated contribution guide --- CONTRIBUTING.md | 97 ++++++++++++++++++++++++++++++++++--------------- MANIFEST.in | 1 + 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a2f93a4..a580cea6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,41 +1,80 @@ -Contributing to UrbanSim -======================== +Thanks for using UrbanSim! -Style ------ +This is an open source project that's part of the Urban Data Science Toolkit. Development and maintenance is a collaboration between UrbanSim Inc, U.C. Berkeley's Urban Analytics Lab, and other contributors. -- Python code should follow the [PEP 8 Style Guide][pep8]. -- Python docstrings should follow the [NumPy documentation format][numpydoc]. +You can contact Sam Maurer, the lead maintainer, at `maurer@urbansim.com`. -### Imports -Imports should be one per line. -Imports should be grouped into standard library, third-party, -and intra-library imports. `from` import should follow "regular" `imports`. -Within each group the imports should be alphabetized. -Here's an example: +## If you have a problem: -```python -import sys -from glob import glob +- Take a look at the [open issues](https://github.com/UDST/urbansim/issues) and [closed issues](https://github.com/UDST/urbansim/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion -import numpy as np +- Open a new issue describing the problem -- if possible, include any error messages, the operating system and version of python you're using, and versions of any libraries that may be relevant -import urbansim.urbansim.modelcompile as modelcompile -from urbansim.util import misc -``` -Imports of scientific Python libraries should follow these conventions: +## Feature proposals: -```python -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import scipy as sp -``` +- Take a look at the [open issues](https://github.com/UDST/urbansim/issues) and [closed issues](https://github.com/UDST/urbansim/issues?q=is%3Aissue+is%3Aclosed) to see if there's already a related discussion +- Post your proposal as a new issue, so we can discuss it (some proposals may not be a good fit for the project) -Thanks! +- Please note that active development of certain UrbanSim components has moved to stand-alone libraries in UDST: Developer, Choicemodels, UrbanSim Templates -[pep8]: http://legacy.python.org/dev/peps/pep-0008/ -[numpydoc]: https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt + +## Contributing code: + +- Create a new branch of `UDST/urbansim`, or fork the repository to your own account + +- Make your changes, following the existing styles for code and inline documentation + +- Add [tests](https://github.com/UDST/urbansim/tree/master/urbansim/tests) if possible! + +- Open a pull request to the `UDST/urbansim` dev branch, including a writeup of your changes -- take a look at some of the closed PR's for examples + +- Current maintainers will review the code, suggest changes, and hopefully merge it! + + +## Updating the documentation: + +- See instructions in `docs/README.md` + + +## Preparing a release: + +- Make a new branch for release prep + +- Update the version number and changelog + - `CHANGELOG.md` + - `setup.py` + - `urbansim/__init__.py` + - `docs/source/index.rst` + +- Make sure all the tests are passing, and check if updates are needed to `README.md` or to the documentation + +- Open a pull request to the master branch to finalize it + +- After merging, tag the release on GitHub and follow the distribution procedures below + + +## Distributing a release on PyPI (for pip installation): + +- Register an account at https://pypi.org, ask one of the current maintainers to add you to the project, and `pip install twine` + +- Check out the copy of the code you'd like to release + +- Run `python setup.py sdist bdist_wheel --universal` + +- This should create a `dist` directory containing two package files -- delete any old ones before the next step + +- Run `twine upload dist/*` -- this will prompt you for your pypi.org credentials + +- Check https://pypi.org/project/urbansim/ for the new version + + +## Distributing a release on Conda Forge (for conda installation): + +- The [conda-forge/urbansim-feedstock](https://github.com/conda-forge/urbansim-feedstock) repository controls the Conda Forge release + +- Conda Forge bots usually detect new releases on PyPI and set in motion the appropriate feedstock updates, which a current maintainer will need to approve and merge + +- Check https://anaconda.org/conda-forge/urbansim for the new version (may take a few minutes for it to appear) diff --git a/MANIFEST.in b/MANIFEST.in index ef85ef0c..9696a5cd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include CHANGELOG.rst +include CONTRIBUTING.md include LICENSE.txt include README.rst include requirements-dev.txt \ No newline at end of file From 5d934c6ff7662dee909cb1c5c80b83d385bae1e7 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 15:21:14 -0700 Subject: [PATCH 43/46] Fixing a numpy deprecation --- urbansim/utils/yamlio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urbansim/utils/yamlio.py b/urbansim/utils/yamlio.py index 476e29f8..2a7ff78a 100644 --- a/urbansim/utils/yamlio.py +++ b/urbansim/utils/yamlio.py @@ -84,7 +84,7 @@ def to_scalar_safe(obj): Convert a numpy data type to a standard python scalar. """ try: - return np.asscalar(obj) + return obj.item() except Exception: return obj From 4131eb37bf812f63693b14002a232801649d52bf Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 15:40:43 -0700 Subject: [PATCH 44/46] Updating readme --- README.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index d16edf34..212871a2 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ How it works Operational UrbanSim models begin with detailed data about a particular region, and then estimate and validate a system of interconnected model components. Full models draw on a number of libraries: not just ``urbansim``, but also `Orca `__ for task orchestration, `Synthpop `__ for population synthesis, `Pandana `__ for network analysis, and so on. Collectively, these make up the `Urban Data Science Toolkit `__ (UDST). -UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by UrbanSim Inc. +UrbanSim models are used by public agencies, consultancies, and researchers in dozens of cities around the U.S. and world. The core platform is open source, but many operational models make use of additional cloud-hosted model building and visualization tools provided by `UrbanSim Inc. `__ Learn More ---------- @@ -43,8 +43,10 @@ Installation Technical documentation ----------------------- -* `Documentation `__ for the ``urbansim`` library +* `Getting started `__ -* `Urban Data Science Toolkit `__ +* `Full documentation `__ -* Documentation for the `UrbanSim Cloud Platform `__ +* Other `UDST `__ libraries + +* Documentation for `UrbanCanvas `__, the UrbanSim cloud platform From 80225d4f41126fe1e5292360b58f3e5e8343fa87 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 5 May 2020 15:42:51 -0700 Subject: [PATCH 45/46] Updating udst link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 212871a2..dd50259c 100644 --- a/README.rst +++ b/README.rst @@ -47,6 +47,6 @@ Technical documentation * `Full documentation `__ -* Other `UDST `__ libraries +* Other `UDST `__ libraries * Documentation for `UrbanCanvas `__, the UrbanSim cloud platform From 0db75668ada0005352b7c7e0a405265f78ccadd7 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 11 May 2020 09:59:38 -0700 Subject: [PATCH 46/46] Typo in docs --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 49663e49..eba9d8a0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,7 @@ UrbanSim UrbanSim is a platform for building statistical models of cities and regions. These models help forecast long-range patterns in real estate development, demographics, and related outcomes, under various policy scenarios. -This ``urbansim``` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. +This ``urbansim`` Python library is a core component. It contains tools for statistical estimation and simulation; domain-specific logic about housing markets, household relocation, and other processes; and frameworks and utilities for assembling a model. v3.2, released May 5, 2020