Merge remote-tracking branch 'repo_org/master' into fix_import_format…

…_pr2 * repo_org/master: DOC: Add docstring validations for "See Also" section (pandas-dev#23143) TST: Fix test assertion (pandas-dev#23357) BUG: Handle Period in combine (pandas-dev#23350) REF: SparseArray imports (pandas-dev#23329) CI: Migrate some CircleCI jobs to Azure (pandas-dev#22992) DOC: update the is_month_start/is_month_end docstring (pandas-dev#23051) Partialy fix issue pandas-dev#23334 - isort pandas/core/groupby directory (pandas-dev#23341) TST: Add base test for extensionarray setitem pandas-dev#23300 (pandas-dev#23304) API: Add sparse Acessor (pandas-dev#23183) PERF: speed up CategoricalIndex.get_loc (pandas-dev#23235)
thoo · Oct 26, 2018 · ebe9fd7 · ebe9fd7
2 parents 0a6218b + f662c5f
commit ebe9fd7
Show file tree

Hide file tree

Showing 48 changed files with 922 additions and 468 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,39 +1,6 @@
 version: 2
 jobs:
 
-  # --------------------------------------------------------------------------
-  # 0. py27_compat
-  # --------------------------------------------------------------------------
-  py27_compat:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-    environment:
-      JOB: "2.7_COMPAT"
-      ENV_FILE: "ci/circle-27-compat.yaml"
-      LOCALE_OVERRIDE: "it_IT.UTF-8"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
   # --------------------------------------------------------------------------
   # 1. py36_locale
   # --------------------------------------------------------------------------
@@ -62,86 +29,14 @@ jobs:
       - run:
           name: build
           command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
+            ./ci/circle/install_circle.sh
+            ./ci/circle/show_circle.sh
       - run:
           name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
-  # --------------------------------------------------------------------------
-  # 2. py36_locale_slow
-  # --------------------------------------------------------------------------
-  py36_locale_slow:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-
-    environment:
-      JOB: "3.6_LOCALE_SLOW"
-      ENV_FILE: "ci/circle-36-locale_slow.yaml"
-      LOCALE_OVERRIDE: "zh_CN.UTF-8"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --only-slow --skip-network
-
-  # --------------------------------------------------------------------------
-  # 3. py35_ascii
-  # --------------------------------------------------------------------------
-  py35_ascii:
-    docker:
-      - image: continuumio/miniconda:latest
-      # databases configuration
-      - image: circleci/postgres:9.6.5-alpine-ram
-        environment:
-            POSTGRES_USER: postgres
-            POSTGRES_DB: pandas_nosetest
-      - image: circleci/mysql:8-ram
-        environment:
-          MYSQL_USER: "root"
-          MYSQL_HOST: "localhost"
-          MYSQL_ALLOW_EMPTY_PASSWORD: "true"
-          MYSQL_DATABASE: "pandas_nosetest"
-
-    environment:
-      JOB: "3.5_ASCII"
-      ENV_FILE: "ci/circle-35-ascii.yaml"
-      LOCALE_OVERRIDE: "C"
-      MINICONDA_DIR: /home/ubuntu/miniconda3
-    steps:
-      - checkout
-      - run:
-          name: build
-          command: |
-            ./ci/install_circle.sh
-            ./ci/show_circle.sh
-      - run:
-          name: test
-          command: ./ci/run_circle.sh  --skip-slow --skip-network
-
+          command: ./ci/circle/run_circle.sh  --skip-slow --skip-network
 
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - py27_compat
       - py36_locale
-      - py36_locale_slow
-      - py35_ascii
diff --git a/.travis.yml b/.travis.yml
@@ -116,10 +116,10 @@ after_success:
 after_script:
   - echo "after_script start"
   - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
-  - if [ -e /tmp/single.xml ]; then
-    ci/print_skipped.py /tmp/single.xml;
+  - if [ -e test-data-single.xml ]; then
+    ci/print_skipped.py test-data-single.xml;
     fi
-  - if [ -e /tmp/multiple.xml ]; then
-    ci/print_skipped.py /tmp/multiple.xml;
+  - if [ -e test-data-multiple.xml ]; then
+    ci/print_skipped.py test-data-multiple.xml;
     fi
   - echo "after_script done"
diff --git a/asv_bench/benchmarks/indexing_engines.py b/asv_bench/benchmarks/indexing_engines.py
@@ -1,18 +1,30 @@
 import numpy as np
 
-from pandas._libs.index import (Int64Engine, UInt64Engine, Float64Engine,
-                                ObjectEngine)
+from pandas._libs import index as libindex
+
+
+def _get_numeric_engines():
+    engine_names = [
+        ('Int64Engine', np.int64), ('Int32Engine', np.int32),
+        ('Int16Engine', np.int16), ('Int8Engine', np.int8),
+        ('UInt64Engine', np.uint64), ('UInt32Engine', np.uint32),
+        ('UInt16engine', np.uint16), ('UInt8Engine', np.uint8),
+        ('Float64Engine', np.float64), ('Float32Engine', np.float32),
+    ]
+    return [(getattr(libindex, engine_name), dtype)
+            for engine_name, dtype in engine_names
+            if hasattr(libindex, engine_name)]
 
 
 class NumericEngineIndexing(object):
 
-    params = [[Int64Engine, UInt64Engine, Float64Engine],
-              [np.int64, np.uint64, np.float64],
+    params = [_get_numeric_engines(),
               ['monotonic_incr', 'monotonic_decr', 'non_monotonic'],
               ]
-    param_names = ['engine', 'dtype', 'index_type']
+    param_names = ['engine_and_dtype', 'index_type']
 
-    def setup(self, engine, dtype, index_type):
+    def setup(self, engine_and_dtype, index_type):
+        engine, dtype = engine_and_dtype
         N = 10**5
         values = list([1] * N + [2] * N + [3] * N)
         arr = {
@@ -26,7 +38,7 @@ def setup(self, engine, dtype, index_type):
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc(2)
 
-    def time_get_loc(self, engine, dtype, index_type):
+    def time_get_loc(self, engine_and_dtype, index_type):
         self.data.get_loc(2)
 
 
@@ -44,7 +56,7 @@ def setup(self, index_type):
             'non_monotonic': np.array(list('abc') * N, dtype=object),
         }[index_type]
 
-        self.data = ObjectEngine(lambda: arr, len(arr))
+        self.data = libindex.ObjectEngine(lambda: arr, len(arr))
         # code belows avoids populating the mapping etc. while timing.
         self.data.get_loc('b')
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -7,10 +7,10 @@ jobs:
   parameters:
     name: macOS
     vmImage: xcode9-macos10.13
-# - template: ci/azure/linux.yml
-#   parameters:
-#     name: Linux
-#     vmImage: ubuntu-16.04
+- template: ci/azure/linux.yml
+  parameters:
+    name: Linux
+    vmImage: ubuntu-16.04
 
 # Windows Python 2.7 needs VC 9.0 installed, and not sure
 # how to make that a conditional task, so for now these are

diff --git a/ci/circle-27-compat.yaml → ci/azure-27-compat.yaml b/ci/circle-27-compat.yaml → ci/azure-27-compat.yaml
diff --git a/ci/circle-36-locale_slow.yaml → ci/azure-36-locale_slow.yaml b/ci/circle-36-locale_slow.yaml → ci/azure-36-locale_slow.yaml
diff --git a/ci/azure-37-locale.yaml b/ci/azure-37-locale.yaml
@@ -0,0 +1,35 @@
+name: pandas
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4
+  - cython>=0.28.2
+  - html5lib
+  - ipython
+  - jinja2
+  - lxml
+  - matplotlib
+  - nomkl
+  - numexpr
+  - numpy
+  - openpyxl=2.5.5
+  - psycopg2
+  - pymysql
+  - pytables
+  - python-dateutil
+  - python=3.6*
+  - pytz
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  # universal
+  - pytest
+  - pytest-xdist
+  - moto
+  - pip:
+    - hypothesis>=3.58.0
diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml
@@ -0,0 +1,56 @@
+parameters:
+  name: ''
+  vmImage: ''
+
+jobs:
+- job: ${{ parameters.name }}
+  pool:
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      py27_np_19:
+        ENV_FILE: ci/azure-27-compat.yaml
+        CONDA_PY: "27"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--skip-slow --skip-network"
+
+      py36_locale:
+        ENV_FILE: ci/azure-37-locale.yaml
+        CONDA_PY: "37"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--skip-slow --skip-network"
+        LOCALE_OVERRIDE: "zh_CN.UTF-8"
+
+      py36_locale_slow:
+        ENV_FILE: ci/azure-36-locale_slow.yaml
+        CONDA_PY: "36"
+        CONDA_ENV: pandas
+        TEST_ARGS: "--only-slow --skip-network"
+
+  steps:
+    - script: |
+        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386; fi
+        echo "Installing Miniconda"{
+        ci/incremental/install_miniconda.sh
+        export PATH=$HOME/miniconda3/bin:$PATH
+        echo "Setting up Conda environment"
+        ci/incremental/setup_conda_environment.sh
+      displayName: 'Before Install'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/incremental/build.sh
+      displayName: 'Build'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        ci/script_single.sh
+        ci/script_multi.sh
+        echo "[Test done]"
+      displayName: 'Test'
+    - script: |
+        export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+    - task: PublishTestResults@2
+      inputs:
+        testResultsFiles: 'test-data-*.xml'
+        testRunTitle: 'Linux'
diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml
@@ -39,5 +39,5 @@ jobs:
         source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
     - task: PublishTestResults@2
       inputs:
-        testResultsFiles: '/tmp/*.xml'
+        testResultsFiles: 'test-data-*.xml'
         testRunTitle: 'MacOS-35'
diff --git a/ci/circle-35-ascii.yaml b/ci/circle-35-ascii.yaml
diff --git a/ci/install_circle.sh → ci/circle/install_circle.sh b/ci/install_circle.sh → ci/circle/install_circle.sh
diff --git a/ci/run_circle.sh → ci/circle/run_circle.sh b/ci/run_circle.sh → ci/circle/run_circle.sh
diff --git a/ci/show_circle.sh → ci/circle/show_circle.sh b/ci/show_circle.sh → ci/circle/show_circle.sh
diff --git a/ci/incremental/setup_conda_environment.sh b/ci/incremental/setup_conda_environment.sh
@@ -27,13 +27,17 @@ set -v
 # w/o removing anything else
 echo
 echo "[removing installed pandas]"
-conda remove pandas -y --force
-pip uninstall -y pandas
+conda remove pandas -y --force || true
+pip uninstall -y pandas || true
 
 echo
 echo "[no installed pandas]"
 conda list pandas
 
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    sudo locale-gen "$LOCALE_OVERRIDE"
+fi
+
 # # Install the compiler toolchain
 # if [[ $(uname) == Linux ]]; then
 #     if [[ "$CONDA_SUBDIR" == "linux-32" || "$BITS32" == "yes" ]] ; then

diff --git a/ci/script_multi.sh b/ci/script_multi.sh
@@ -27,17 +27,17 @@ if [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
 
 elif [ "$COVERAGE" ]; then
-    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
 
 elif [ "$SLOW" ]; then
     TEST_ARGS="--only-slow --skip-network"
-    echo pytest -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -m "not single and slow" -v --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
+    echo pytest -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest      -m "not single and slow" -v --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
 
 else
-    echo pytest -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas
-    pytest      -n 2 -m "not single" --junitxml=/tmp/multiple.xml --strict $TEST_ARGS pandas # TODO: doctest
+    echo pytest -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas
+    pytest      -n 2 -m "not single" --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas # TODO: doctest
 
 fi