Merge pull request #742 from msehnout/remove-keywords-workers

remove keywords tagger and summary tasks
fabric8-analytics · Nov 26, 2018 · cfffcb8 · cfffcb8
2 parents d69c364 + bcdf2fa
commit cfffcb8
Show file tree

Hide file tree

Showing 16 changed files with 1 addition and 703 deletions.
diff --git a/f8a_worker/dispatcher/flows/bayesianAnalysisFlow.yml b/f8a_worker/dispatcher/flows/bayesianAnalysisFlow.yml
@@ -40,10 +40,6 @@
         - from:
           to:
             - 'digests'
-        - from: 'metadata'
-          to: 'keywords_tagging'
-          condition:
-            name: 'alwaysFalse'
         - from: 'metadata'
           to: 'bayesianPriorityPackageFlow'
           condition: &priorityAnalysisCheck

diff --git a/f8a_worker/dispatcher/flows/bayesianPackageFlow.yml b/f8a_worker/dispatcher/flows/bayesianPackageFlow.yml
@@ -5,7 +5,6 @@
       queue: '{DEPLOYMENT_PREFIX}_ingestion_bayesianPackageFlow_v0'
       propagate_node_args:
         - 'bayesianPackageAnalysisFlow'
-        - 'bayesianPackageTaggingFlow'
       node_args_from_first: true
       nowait: 'PackageGraphImporterTask'
       sampling:
@@ -22,18 +21,7 @@
             args:
               key: 'document_id'
         - from: 'bayesianPackageAnalysisFlow'
-          to: 'bayesianPackageTaggingFlow'
-          condition: &shouldSchedulePackageTaggingCheck
-            name: 'alwaysFalse'
-        - from: 'bayesianPackageAnalysisFlow'
-          to: 'PackageFinalizeTask'
-          condition:
-            not:
-              <<: *shouldSchedulePackageTaggingCheck
-        - from: 'bayesianPackageTaggingFlow'
           to: 'PackageFinalizeTask'
-          condition:
-            <<: *shouldSchedulePackageTaggingCheck
         - from: 'PackageFinalizeTask'
           to: 'PackageResultCollector'
         - from: 'PackageResultCollector'
@@ -56,10 +44,6 @@
                   value: true
       failures:
         - nodes: 'bayesianPackageAnalysisFlow'
-          fallback: 'bayesianPackageTaggingFlow'
-          condition:
-            <<: *shouldSchedulePackageTaggingCheck
-        - nodes: 'bayesianPackageTaggingFlow'
           fallback: 'PackageFinalizeTaskError'
 
     - <<: *bayesianPackageFlow

diff --git a/f8a_worker/dispatcher/flows/bayesianPackageTaggingFlow.yml b/f8a_worker/dispatcher/flows/bayesianPackageTaggingFlow.yml
diff --git a/f8a_worker/dispatcher/flows/keywordsSummaryFlow.yml b/f8a_worker/dispatcher/flows/keywordsSummaryFlow.yml
diff --git a/f8a_worker/dispatcher/nodes.yml b/f8a_worker/dispatcher/nodes.yml
@@ -150,32 +150,11 @@
       import: 'f8a_worker.workers'
       max_retry: 0
       queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_BookkeeperTask_v0'
-    - name: 'package_keywords_tagging'
-      classname: 'PackageKeywordsTaggingTask'
-      storage_task_name: 'keywords_tagging'
-      import: 'f8a_worker.workers'
-      max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_package_keywords_tagging_v0'
-      storage: 'PackagePostgres'
-      selective_run_function:
-        name: 'selective_run_function'
-        import: 'f8a_worker.dispatcher.selective'
-    - name: 'keywords_tagging'
-      classname: 'KeywordsTaggingTask'
-      import: 'f8a_worker.workers'
-      max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_keywords_tagging_v0'
-      storage: 'BayesianPostgres'
     - name: 'RepositoryDescCollectorTask'
       import: 'f8a_worker.workers'
       max_retry: 0
       queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_RepositoryDescCollectorTask_v0'
       storage: 'S3RepositoryDescription'
-    - name: 'KeywordsSummaryTask'
-      import: 'f8a_worker.workers'
-      max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_KeywordsSummaryTask_tagging_v0'
-      storage: 'S3KeywordsSummary'
     - name: 'GithubDependencyTreeTask'
       classname: 'GithubDependencyTreeTask'
       import: 'f8a_worker.workers'
@@ -218,11 +197,7 @@
     - 'bayesianApiPackageFlow'
     - 'bayesianPriorityPackageFlow'
     - 'bayesianPackageAnalysisFlow'
-    - 'bayesianPackageTaggingFlow'
     - 'componentApiFlow'
-    - 'keywordsSummaryFlow'
-    - 'keywordsApiSummaryFlow'
-    - 'keywordsPrioritySummaryFlow'
     - 'dependencyIngestFlow'
     - 'osioUserNotificationFlow'
 
@@ -329,15 +304,6 @@
         encryption: false
         versioned: true
 
-    - name: 'S3KeywordsSummary'
-      import: 'f8a_worker.storages'
-      configuration:
-        <<: *configurationS3
-        bucket_name: '{DEPLOYMENT_PREFIX}-keywords-summary'
-        region_name: 'us-east-1'
-        encryption: false
-        versioned: true
-
     - name: 'S3UserIntent'
       import: 'f8a_worker.storages'
       configuration:

diff --git a/f8a_worker/dispatcher/selective.py b/f8a_worker/dispatcher/selective.py
@@ -17,8 +17,7 @@ def selective_run_function(flow_name, node_name, node_args, task_names, storage_
     :return: ID of task that should be reused, None if task should be run again
     """
     try:
-        if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow',
-                         'bayesianPackageTaggingFlow'):
+        if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow'):
             task_result = storage_pool.get_connected_storage('PackagePostgres').\
                 get_latest_task_entry(
                     node_args['ecosystem'],

diff --git a/f8a_worker/storages/__init__.py b/f8a_worker/storages/__init__.py
@@ -15,7 +15,6 @@
 from .s3_gh_manifests import S3GitHubManifestMetadata
 from .s3_userprofilestore import S3UserProfileStore
 from .s3_description_repository import S3RepositoryDescription
-from .s3_keywords_summary import S3KeywordsSummary
 from .s3_userintent import S3UserIntent
 from .s3_manual_tagging import S3ManualTagging
 from .s3_crowd_source_tags import S3CrowdSourceTags
@@ -34,7 +33,6 @@
 assert S3GitHubManifestMetadata is not None
 assert S3UserProfileStore is not None
 assert S3RepositoryDescription is not None
-assert S3KeywordsSummary is not None
 assert S3UserIntent is not None
 assert S3ManualTagging is not None
 assert S3CrowdSourceTags is not None
diff --git a/f8a_worker/storages/s3_keywords_summary.py b/f8a_worker/storages/s3_keywords_summary.py
diff --git a/f8a_worker/workers/__init__.py b/f8a_worker/workers/__init__.py
@@ -12,9 +12,6 @@
 from f8a_worker.workers.graphaggregator import GraphAggregatorTask
 from f8a_worker.workers.init_analysis_flow import InitAnalysisFlow
 from f8a_worker.workers.init_package_flow import InitPackageFlow
-from f8a_worker.workers.keywords_summary import KeywordsSummaryTask
-from f8a_worker.workers.keywords_tagging import KeywordsTaggingTask
-from f8a_worker.workers.keywords_tagging import PackageKeywordsTaggingTask
 from f8a_worker.workers.libraries_io import LibrariesIoTask
 from f8a_worker.workers.license import LicenseCheckTask
 from f8a_worker.workers.mercator import MercatorTask
@@ -38,9 +35,6 @@
 assert GraphAggregatorTask is not None
 assert InitAnalysisFlow is not None
 assert InitPackageFlow is not None
-assert KeywordsSummaryTask is not None
-assert KeywordsTaggingTask is not None
-assert PackageKeywordsTaggingTask is not None
 assert LibrariesIoTask is not None
 assert LicenseCheckTask is not None
 assert MercatorTask is not None

diff --git a/f8a_worker/workers/keywords_summary.py b/f8a_worker/workers/keywords_summary.py