Restructure project layout

.github/workflows/ci.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install latest Rust nightly
+      - name: Install supported Rust nightly
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
@@ -24,10 +24,6 @@ jobs:
           components: rustfmt, rust-src
       - name: Check Rust formatting
         run: cargo fmt -- --check
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
       - name: Build
         run: cargo build --verbose
       - name: Run tests
@@ -38,7 +34,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v2
       - name: Setup python
@@ -47,19 +43,19 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install Python dependencies
         run: |
-          python -m pip install setuptools_rust flake8 yapf flake8-quotes
+          python -m pip install maturin flake8 yapf==0.31.0 flake8-quotes
       - name: Check Python formatting
         run: |
-          flake8 --inline-quotes="double" ./doc/
+          flake8 --inline-quotes="double" ./python/doc/
           flake8 --inline-quotes="double" ./python/
-          yapf --diff --recursive ./doc/
+          yapf --diff --recursive ./python/doc/
           yapf --diff --recursive ./python/
-      - name: Install latest Rust nightly
+      - name: Install supported Rust nightly
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
+          profile: minimal
           override: true
-          components: rustfmt
+          components: rustfmt, rust-src
       - name: Build
-        # Building with develop is faster than with install
-        run: python python/setup.py develop
+        run: cd python && maturin build

.github/workflows/docs.yaml

@@ -12,19 +12,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install latest Rust nightly
+      - name: Install supported Rust nightly
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
           profile: minimal
           override: true
           components: rustfmt, rust-src
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - name: Install Python dependencies
-        run: python -m pip install fire
       - name: Build Documentation
         run: cargo doc --all --no-deps
 
@@ -35,10 +29,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6
+          python-version: 3.x
       - name: Install Python dependencies
-        run: python -m pip install -r doc/requirements-doc.txt
+        run: python -m pip install -r python/doc/requirements-doc.txt
       - name: Build Documentation
         run: |
-          cd doc
-          make html
+          cd python/doc
+          make html SPHINXOPTS="-W --keep-going -n"

.gitignore

@@ -11,7 +11,7 @@ Cargo.lock
 # Python files
 *.egg-info
 __pycache__
-/env
+env
 *.pyc
 dist
 /build
@@ -22,7 +22,7 @@ dist
 *~
 
 # Generated documentation files
-/doc/build
+/python/doc/build
 .idea/
 venv/
-/doc/source/_build
+/python/doc/source/_build

Cargo.toml

@@ -1,60 +1,6 @@
-[package]
-name = "erdos"
-version = "0.3.1"
-authors = ["The ERDOS Team"]
-edition = "2018"
-default-run = "erdos"
-license = "Apache-2.0"
-readme = "README.md"
-documentation = "https://docs.rs/crate/erdos/"
-repository = "https://github.com/erdos-project/erdos"
-description = """
-ERDOS is a platform for developing self-driving cars and robotics applications.
-"""
-keywords = ["data-flow", "robotics", "autonomos", "driving"]
+[workspace]
 
-[dependencies]
-abomonation = "0.7.3"
-abomonation_derive = "0.5.0"
-async-trait = "0.1.18"
-bincode = "1.3.1"
-bytes = "1.0.0"
-byteorder = "1.3.4"
-clap = "2.33.0"
-futures = "0.3.5"
-futures-delay-queue = "0.4.2"
-futures-intrusive = "0.4"  # Required for futures-delay-queue
-futures-util = "0.3.5"
-petgraph = "0.5.0"
-pyo3 = { version = "0.14.3", optional = true }
-rand = "0.3"
-rosrust = { version = "0.9", optional = true }
-rosrust_msg = { version = "0.1", optional = true }
-serde = { version = "1.0.115", features = ["derive"] }
-tracing = "0.1.29"
-tracing-appender = "0.2.0"
-tracing-subscriber = "0.3.1"
-tokio = { version = "1.8.1", features = ["full"] }
-tokio-util = { version = "0.6.7", features = ["codec"] }
-tokio-serde-bincode = "0.2.1"
-uuid = { version = "0.7", features = ["v4", "v5", "serde"] }
-
-[build-dependencies]
-slog = "2.4.2"
-slog-term = "2.4.2"
-
-[dev-dependencies]
-criterion = "0.3.3"
-criterion-macro = "0.3.3"
-nix = "0.17.0"
-
-[features]
-default = []
-python = ["pyo3"]  # Target python with 'cargo build --features=python
-ros = ["rosrust", "rosrust_msg"]  # Add ROS operator support with 'cargo build --features=ros
-
-[lib]
-crate-type=["rlib", "cdylib"]   # Required for python
-
-[[bench]]
-name = "latency"
+members = [
+    "erdos",
+    "python",
+]

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Set up an erdos user first.
 RUN apt-get -y update && apt-get -y install sudo
@@ -18,18 +18,20 @@ ENV SHELL /bin/bash
 
 SHELL ["/bin/bash", "-c"]
 
-# Instal rust.
+# Install rust.
 RUN sudo apt-get -y install curl clang python3 python3-pip git
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/home/erdos/.cargo/bin:${PATH}"
-RUN rustup default nightly
 
 # Get the erdos directory.
-RUN pip3 install setuptools setuptools-rust
 RUN mkdir -p /home/erdos/workspace
 RUN cd /home/erdos/workspace && git clone https://github.com/erdos-project/erdos.git
 WORKDIR /home/erdos/workspace/erdos
 RUN cd /home/erdos/workspace/erdos
+
+# Install erdos.
 RUN cargo build --release
 # Install the python package
-RUN python3 python/setup.py install --user
+ENV PATH="/home/erdos/.local/bin:${PATH}"
+RUN pip3 install -U pip>=21.3
+RUN cd python && pip3 install -e .

README.md

@@ -35,7 +35,6 @@ If you'd like to contribute to ERDOS, first
 [install Rust](https://www.rust-lang.org/tools/install).
 Then run the following to clone the repository and build ERDOS:
 ```console
-rustup default nightly  # use nightly Rust toolchain
 git clone https://github.com/erdos-project/erdos.git && cd erdos
 cargo build
 ```
@@ -48,34 +47,26 @@ To develop an ERDOS application in Python, simply run
 
 If you'd like to contribute to ERDOS, first
 [install Rust](https://www.rust-lang.org/tools/install).
-Then run the following to clone the repository and build ERDOS:
+Within a [virtual environment](https://docs.python.org/3/tutorial/venv.html),
+run the following to clone the repository and build ERDOS:
 ```console
-rustup default nightly  # use nightly Rust toolchain
-git clone https://github.com/erdos-project/erdos.git && cd erdos
-python3 python/setup.py develop
+git clone https://github.com/erdos-project/erdos.git && cd erdos/python
+pip3 install maturin
+maturin develop
 ```
 
-Python files are available under the `python/` directory, and the Python-Rust
-bridge interface is developed under `src/python/`.
+The Python-Rust bridge interface is developed in the `python` crate, which
+also contains user-facing python files under the `python/erdos` directory.
 
 If you'd like to build ERDOS for release (has better performance, but longer
-build times), run `python3 python/setup.py install`.
+build times), run `maturin develop --release`.
 
 ## Running an example
 
 ```console
 python3 python/examples/simple_pipeline.py
 ```
 
-## Build parameters
-
-By default, ERDOS supports up to 20 read streams and 10 write streams in a stream bundle (used to add callbacks across multiple streams; see callback builder in Rust docs for more details).
-Some applications may require bundles that support more read and write streams. This can be configured by setting the `ERDOS_BUNDLE_MAX_READ_STREAMS` and `ERDOS_BUNDLE_MAX_WRITE_STREAMS` environment variables when building.
-
-Building ERDOS for the first time may be slow because these parameters result in generated code which dominates the build time.
-Subsequent builds should be much faster due to caching unless `build.rs` or the code generation scripts are modified.
-In that case, consider setting `ERDOS_BUNDLE_MAX_READ_STREAMS` and `ERDOS_BUNDLE_MAX_WRITE_STREAMS` to speed up builds.
-
 # Writing Applications
 
 ERDOS provides Python and Rust interfaces for developing applications.
@@ -112,7 +103,7 @@ second of data on small clusters. Therefore, ERDOS is optimized to
 send small amounts of data (gigabytes as opposed to terabytes)
 as quickly as possible.
 
-ERDOS provides determinisim through **watermarks**. Low watermarks
+ERDOS provides determinism through **watermarks**. Low watermarks
 are a bound on the age of messages received and operators will ignore
 any messages older than the most recent watermark received. By processing
 on watermarks, applications can avoid non-determinism from processing