Add more scripts

PROFILES

@@ -2,10 +2,12 @@ catalan/ca-es_pocketsphinx-cmu
 dutch/nl_kaldi-cgn
 dutch/nl_pocketsphinx-cmu
 english/en-in_pocketsphinx-cmu
+english/en-us_deepspeech-mozilla
 english/en-us_julius-github
 english/en-us_kaldi-zamia
 english/en-us_pocketsphinx-cmu
 french/fr_pocketsphinx-cmu
+german/de_deepspeech-aashishag
 german/de_kaldi-zamia
 german/de_pocketsphinx-cmu
 greek/el-gr_pocketsphinx-cmu

__main__.py

@@ -41,4 +41,4 @@ async def download_raw(path: str) -> Response:
 # -----------------------------------------------------------------------------
 
 if __name__ == "__main__":
-    app.run()
+    app.run(host="0.0.0.0")

bin/check_files_yml.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Verifies sizes and sha256 sums for files.yml files."""
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+
+def main():
+    """Main entry point"""
+    for files_yaml_path in sys.argv[1:]:
+        profile_root = Path(files_yaml_path).parent
+        with open(files_yaml_path, "r") as files_yaml_file:
+            files_yaml = yaml.safe_load(files_yaml_file)
+
+        file_count = 0
+        for condition, files in files_yaml.items():
+            for file_path, file_info in files.items():
+                full_path = profile_root / file_path
+
+                # Check byte size
+                expected_bytes = int(file_info["bytes"])
+                actual_bytes = os.path.getsize(full_path)
+
+                assert (
+                    actual_bytes == expected_bytes
+                ), f"Expected size of {full_path} to be {expected_bytes}, got {actual_bytes}"
+
+                # Check sha256 sum
+                expected_sum = str(file_info["sha256"]).strip()
+                sum_result = subprocess.check_output(["sha256sum", str(full_path)]).decode().strip()
+                actual_sum = sum_result.split()[0]
+
+                assert (
+                    actual_sum == expected_sum
+                ), f"Expected sha256 sum of {full_path} to be {expected_sum}, got {actual_sum}"
+
+                file_count += 1
+
+        print(profile_root.name, file_count, "OK")
+
+
+if __name__ == "__main__":
+    main()

bin/generate-reports.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -e
+
+this_dir="$( cd "$( dirname "$0" )" && pwd )"
+src_dir="$(realpath "${this_dir}/..")"
+
+# -----------------------------------------------------------------------------
+
+# Create a temporary directory for testing
+temp_dir="$(mktemp -d)"
+
+function cleanup {
+    rm -rf "${temp_dir}"
+}
+
+trap cleanup EXIT
+
+# -----------------------------------------------------------------------------
+
+profiles=()
+while [[ ! -z "$1" ]]; do
+    profiles+=("$1")
+    shift
+done
+
+if [[ -z "${profiles[*]}" ]]; then
+    while read -r profile; do
+        profiles+=("${profile}")
+    done < "${src_dir}/PROFILES"
+fi
+
+# -----------------------------------------------------------------------------
+
+for profile in "${profiles[@]}"; do
+    echo "${profile}"
+
+    # Copy to temporary directory
+    dest_dir="${temp_dir}/${profile}"
+    rm -rf "${dest_dir}"
+    mkdir -p "${dest_dir}"
+
+    echo 'Copying...'
+    cp -aR "${src_dir}/${profile}"/* "${dest_dir}/"
+
+    echo 'Training...'
+    voice2json -p "${dest_dir}" --debug train-profile
+
+    closed_dir="${dest_dir}/test/closed"
+    if [[ -d "${closed_dir}" ]]; then
+        echo 'Testing (closed)...'
+        voice2json -p "${dest_dir}" --debug test-examples --directory "${closed_dir}" | \
+            jq . > "${closed_dir}/report.json"
+
+        cp "${closed_dir}/report.json" "${src_dir}/${profile}/test/closed/"
+    fi
+
+    open_dir="${dest_dir}/test/open"
+    if [[ -d "${open_dir}" ]]; then
+        echo 'Testing (open)...'
+        voice2json -p "${dest_dir}" --debug test-examples --open --directory "${open_dir}" | \
+            jq . > "${open_dir}/report.json"
+
+        cp "${open_dir}/report.json" "${src_dir}/${profile}/test/open/"
+    fi
+
+    echo 'Done'
+    echo '----------'
+    echo ''
+done

bin/make_download_json.py

@@ -58,7 +58,7 @@ def main():
                     "bytes_expected": os.path.getsize(file_name),
                 }
             )
-            file_details["parts"] = parts
+            file_details["parts"] = sorted(parts, key=lambda p: p["fragment"])
         elif unzip:
             file_details["url"] += ".gz"
 

bin/update_files_yml.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""Update sizes and sha256 sums for files.yml files."""
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+
+def main():
+    """Main entry point"""
+    assert len(sys.argv) > 1
+    files_yaml_path = sys.argv[1]
+    profile_root = Path(files_yaml_path).parent
+    with open(files_yaml_path, "r") as files_yaml_file:
+        files_yaml = yaml.safe_load(files_yaml_file)
+
+    file_count = 0
+    for condition, files in files_yaml.items():
+        for file_path, file_info in files.items():
+            full_path = profile_root / file_path
+
+            file_info["bytes"] = os.path.getsize(full_path)
+            sum_result = subprocess.check_output(["sha256sum", str(full_path)]).decode().strip()
+            file_info["sha256"] = sum_result.split()[0]
+
+    print(yaml.dump(files_yaml))
+
+
+if __name__ == "__main__":
+    main()