Clean up progress

.devcontainer/Dockerfile

@@ -6,11 +6,11 @@ WORKDIR /home/vscode
 # Install python environment manager
 RUN curl https://pyenv.run | bash && \
     echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bashrc && \
-    echo 'eval "$($HOME/.pyenv/bin/pyenv init -)"' >> ~/.bashrc && \
     echo 'eval "$($HOME/.pyenv/bin/pyenv init -)"' >> ~/.bashrc
 
 # Install & setup python environment
-# env PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto --enable-shared' PYTHON_CFLAGS='-march=native -mtune=native'
+ENV PYTHON_CONFIGURE_OPTS='--enable-shared'
+#ENV PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto --enable-shared' PYTHON_CFLAGS='-march=native -mtune=native'
 RUN $HOME/.pyenv/bin/pyenv install 3.9.16  && \
     $HOME/.pyenv/bin/pyenv virtualenv 3.9.16 dev && \
     $HOME/.pyenv/bin/pyenv global 3.9.16  && \
@@ -20,7 +20,7 @@ RUN $HOME/.pyenv/bin/pyenv install 3.9.16  && \
 # Clone the skypie-precomputation repo
 RUN git clone https://github.com/hydro-project/cloud_oracle_precomputer.git
 
-WORKDIR /home/vscode/cloud_oracle_precomputer
+WORKDIR $HOME/cloud_oracle_precomputer
 
 #RUN git checkout dev
 
@@ -38,15 +38,15 @@ RUN $HOME/.pyenv/shims/pip install -r Requirements.txt
 RUN $HOME/.pyenv/shims/pip install -e baselines && \
     $HOME/.pyenv/shims/pip install -e proto_messages
 
-WORKDIR /home/vscode
+WORKDIR $HOME
 #RUN sudo rm -rf cloud_oracle_precomputer
 
 # Install SkyPIE oracle
 RUN git clone https://github.com/hydro-project/cloud_oracle_skypie.git
-WORKDIR /home/vscode/cloud_oracle_skypie
+WORKDIR $HOME/cloud_oracle_skypie
 RUN $HOME/.pyenv/shims/pip install -e .
 
-WORKDIR /home/vscode
+WORKDIR $HOME
 RUN sudo rm -rf cloud_oracle_skypie
 
 # Warm up the cache for the Rust build

.vscode/launch.json

@@ -4,6 +4,18 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Python Debugger: Precomputer",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "deploy",
+            "args" :[
+                "--experiment", "skystore_no_slos_aws_rep1",
+                "--redundancy-elimination-workers", "1",
+                "--profile", "dev",
+                "--output-dir", "~/local_results"
+            ]
+        },
         {
             "type": "lldb",
             "request": "launch",
@@ -19,7 +31,8 @@
                     "kind": "bin"
                 }
             },
-            "args": ["-r=aws",
+            "args": [
+                "-r=aws",
                 "-n=network_cost_v2.csv",
                 "-s=storage_pricing.csv",
                 "-b=200",

Requirements.txt

@@ -1,2 +1,3 @@
 hydro-deploy==0.5.0
+maturin
 ./python_redundancy_elimination

deploy/__main__.py

@@ -17,7 +17,7 @@ def get_args(args):
     precomp_args.add_argument("--batch-size", type=int, help="The batch size to use for the precomputation. Defaults to all available tiers (\'\')")
     precomp_args.add_argument("--hydro-dir", type=str, help="The directory of the SkyPie precomputer hydroflow project.")
     precomp_args.add_argument("--data-dir", type=str, help="The data directory of the supplemental files.")
-    precomp_args.add_argument("--experiment-dir", type=str, help="The base directory to store the experiment results.")
+    precomp_args.add_argument("--output-dir", type=str, help="The base directory to store the experiment results.")
     precomp_args.add_argument("--profile", type=str, help="The compiler profile to use, e.g., dev or release.")
     precomp_args.add_argument("--latency-slo", type=float, help="The latency SLO to use for the precomputation.")
 

deploy/experiment.py

@@ -8,7 +8,7 @@ class Experiment:
     redundancy_elimination_workers: int
     replication_factor_max: int = None
     batch_size: int = 400
-    experiment_dir: str = field(default_factory=lambda: os.path.join(os.getcwd(), "experiments"))
+    output_dir: str = field(default_factory=lambda: os.path.join(os.getcwd(), "experiments"))
     hydro_dir: str = field(default_factory=lambda: os.path.join(os.getcwd(), "skypie_lib"))
     data_dir: str = field(default_factory=lambda: os.path.join(os.getcwd(), "data"))
     profile: str = "release"
@@ -43,7 +43,7 @@ def __post_init__(self):
         paths = ([self.experiment_name] if self.experiment_name is not None else []) + \
             ([friendly_latency_slo] if self.latency_slo is not None else []) + \
             [friendly_region_and_object_store, f"{self.replication_factor}-{self.replication_factor_max or self.replication_factor}", str(self.redundancy_elimination_workers), str(self.batch_size), str(self.optimizer), clarkson]
-        self.experiment_dir_full = os.path.join(self.experiment_dir, *paths)
+        self.experiment_dir_full = os.path.join(self.output_dir, *paths)
 
     def copy(self, **kwargs):
         kwargs = {**self.__dict__, **kwargs}

deploy/experiments/sigmod.py

@@ -1,11 +1,26 @@
 import os
 from deploy import Experiment
 
+def build_real_trace_experiments():
+
+    fixed_args = dict(
+        output_dir = os.path.join(os.getcwd(), "results", "precomputation_real_trace"),
+        batch_size = 200,
+        redundancy_elimination_workers = 80,
+        replication_factor = 1,
+        replication_factor_max = 4,
+        optimizer="PrimalSimplex",
+        use_clarkson=False,
+        #profile= "dev"
+    )
+
+    return [Experiment(region_selector="aws", **fixed_args)]
+
 def build_scaling_experiments():
     min_replication_factor = 1
     max_replication_factor = 5
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling"),
+        output_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling"),
         batch_size = 200,
         redundancy_elimination_workers = 80,
         replication_factor = 0,
@@ -15,10 +30,10 @@ def build_scaling_experiments():
     )
 
     scaling = Experiment(region_selector="aws-eu", object_store_selector="General Purpose", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
-        Experiment(region_selector="aws-eu", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) #+ \
-        #Experiment(region_selector="aws", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
-        #Experiment(region_selector="azure", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
-        #Experiment(region_selector="azure|aws", **fixed_args).as_replication_factors(min_replication_factor, min(4, max_replication_factor))
+        Experiment(region_selector="aws-eu", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
+        Experiment(region_selector="aws", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
+        Experiment(region_selector="azure", **fixed_args).as_replication_factors(min_replication_factor, max_replication_factor) + \
+        Experiment(region_selector="azure|aws", **fixed_args).as_replication_factors(min_replication_factor, min(4, max_replication_factor))
 
     # Order by replication factor
     scaling.sort(key=lambda e: e.replication_factor)
@@ -31,7 +46,7 @@ def build_scaling_experiments():
 
 def build_scaling_experiments_lrs():
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling_lrs"),
+        output_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling_lrs"),
         batch_size = 200,
         redundancy_elimination_workers = 80,
         #redundancy_elimination_workers = 1,
@@ -57,15 +72,15 @@ def build_scaling_experiment_candidates():
     experiments = build_scaling_experiments_lrs()
 
     for e in experiments:
-        e.experiment_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling_small_candidates")
+        e.output_dir = os.path.join(os.getcwd(), "results", "precomputation_scaling_small_candidates")
         e.output_candidates = True
         e.optimizer = "PrimalSimplex"
 
     return experiments
 
 def build_precomputation_batching_experiments(large=False):
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "batch_size_scaling"),
+        output_dir = os.path.join(os.getcwd(), "results", "batch_size_scaling"),
         redundancy_elimination_workers = 80,
         region_selector="aws|azure",
         object_store_selector="",
@@ -78,14 +93,14 @@ def build_precomputation_batching_experiments(large=False):
     batch_sizes = [200, 500, 1000]
 
     if large:
-        fixed_args["experiment_dir"] = os.path.join(os.getcwd(), "results", "batch_size_scaling_large")
+        fixed_args["output_dir"] = os.path.join(os.getcwd(), "results", "batch_size_scaling_large")
         fixed_args["replication_factor"] = 5
 
     return [Experiment(**fixed_args, batch_size=b) for b in batch_sizes]
 
 def build_cpu_scaling_experiments():
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "cpu_scaling"),
+        output_dir = os.path.join(os.getcwd(), "results", "cpu_scaling"),
         #replication_factor=5,
         replication_factor=3,
         batch_size=200,
@@ -102,7 +117,7 @@ def build_cpu_scaling_experiments():
 
 def build_accuracy_small_experiments():
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "accuracy"),
+        output_dir = os.path.join(os.getcwd(), "results", "accuracy"),
         replication_factor=2,
         batch_size=200,
         #optimizer="lrs",
@@ -129,9 +144,10 @@ def build_accuracy_small_experiments():
 
 named_experiments = {
     "scaling": build_scaling_experiments,
-    "scaling_lrs": build_scaling_experiments_lrs,
-    "scaling_candidates": build_scaling_experiment_candidates,
+    "real_trace": build_real_trace_experiments,
+    #"scaling_lrs": build_scaling_experiments_lrs,
+    #"scaling_candidates": build_scaling_experiment_candidates,
     "batch_size": build_precomputation_batching_experiments,
-    "cpu_scaling": build_cpu_scaling_experiments,
-    "accuracy": build_accuracy_small_experiments,
+    #"cpu_scaling": build_cpu_scaling_experiments,
+    #"accuracy": build_accuracy_small_experiments,
 }

deploy/experiments/skystore.py

@@ -4,7 +4,7 @@
 
 def build_skystore_experiments(*, latency_slos=[2.0, 4.0, 8.0], replication_factors=[1,2,3,5,8], region_selectors=None, replication_range: "Tuple[int,int]|None" = None):
     fixed_args = dict(
-        experiment_dir = os.path.join(os.getcwd(), "results", "skystore"),
+        output_dir = os.path.join(os.getcwd(), "results", "skystore"),
         batch_size = 200,
         redundancy_elimination_workers = 80,
         #redundancy_elimination_workers = 1,
@@ -41,4 +41,5 @@ def build_skystore_experiments(*, latency_slos=[2.0, 4.0, 8.0], replication_fact
     "slos_aws": lambda: build_skystore_experiments(latency_slos=[6.0], replication_range=(1,5), region_selectors=["aws"]),
     "no_slos_aws": lambda: build_skystore_experiments(latency_slos=[None], replication_range=(1,5), region_selectors=["aws"]),
     "no_slos_aws_rep1": lambda: build_skystore_experiments(latency_slos=[None], replication_range=(1,1), region_selectors=["aws"]),
+    "no_slos_multicloud_rep1": lambda: build_skystore_experiments(latency_slos=[None], replication_range=(1,1), region_selectors=[""]),
 }