# This is a combination of 17 commits.

# This is the 1st commit message:

Merge pull request fabric8-analytics#742 from msehnout/remove-keywords-workers

remove keywords tagger and summary tasks
# This is the commit message fabric8-analytics#2:

initial commit for git operations addition in worker

# This is the commit message fabric8-analytics#3:

adding logs

# This is the commit message fabric8-analytics#4:

lint

# This is the commit message fabric8-analytics#5:

file rectified

# This is the commit message fabric8-analytics#6:

rectified queue name

# This is the commit message fabric8-analytics#7:

added functionanilty for git operations

# This is the commit message fabric8-analytics#8:

lint

# This is the commit message fabric8-analytics#9:

rectified clone

# This is the commit message fabric8-analytics#10:

additional logic for clone

# This is the commit message fabric8-analytics#11:

token rectified

# This is the commit message fabric8-analytics#12:

gh username rectified

# This is the commit message fabric8-analytics#13:

rectified gemini call

# This is the commit message fabric8-analytics#14:

added logs

# This is the commit message fabric8-analytics#15:

rectified parser logic

# This is the commit message fabric8-analytics#16:

modified logic for gemini

# This is the commit message fabric8-analytics#17:

host resolved

f8a_worker/dispatcher/flows/bayesianAnalysisFlow.yml

@@ -40,10 +40,6 @@
         - from:
           to:
             - 'digests'
-        - from: 'metadata'
-          to: 'keywords_tagging'
-          condition:
-            name: 'alwaysFalse'
         - from: 'metadata'
           to: 'bayesianPriorityPackageFlow'
           condition: &priorityAnalysisCheck

f8a_worker/dispatcher/flows/bayesianPackageFlow.yml

@@ -5,7 +5,6 @@
       queue: '{DEPLOYMENT_PREFIX}_ingestion_bayesianPackageFlow_v0'
       propagate_node_args:
         - 'bayesianPackageAnalysisFlow'
-        - 'bayesianPackageTaggingFlow'
       node_args_from_first: true
       nowait: 'PackageGraphImporterTask'
       sampling:
@@ -22,18 +21,7 @@
             args:
               key: 'document_id'
         - from: 'bayesianPackageAnalysisFlow'
-          to: 'bayesianPackageTaggingFlow'
-          condition: &shouldSchedulePackageTaggingCheck
-            name: 'alwaysFalse'
-        - from: 'bayesianPackageAnalysisFlow'
-          to: 'PackageFinalizeTask'
-          condition:
-            not:
-              <<: *shouldSchedulePackageTaggingCheck
-        - from: 'bayesianPackageTaggingFlow'
           to: 'PackageFinalizeTask'
-          condition:
-            <<: *shouldSchedulePackageTaggingCheck
         - from: 'PackageFinalizeTask'
           to: 'PackageResultCollector'
         - from: 'PackageResultCollector'
@@ -56,10 +44,6 @@
                   value: true
       failures:
         - nodes: 'bayesianPackageAnalysisFlow'
-          fallback: 'bayesianPackageTaggingFlow'
-          condition:
-            <<: *shouldSchedulePackageTaggingCheck
-        - nodes: 'bayesianPackageTaggingFlow'
           fallback: 'PackageFinalizeTaskError'
 
     - <<: *bayesianPackageFlow

f8a_worker/dispatcher/flows/gitOperationsFlow.yml

@@ -0,0 +1,10 @@
+---
+  # This flow is for the git operations to generate the manifest files for the stack analysis.
+  flow-definitions:
+    - name: 'gitOperationsFlow'
+      queue: '{DEPLOYMENT_PREFIX}_api_gitOperationsFlow_v0'
+      propagate_node_args: true
+      node_args_from_first: true
+      edges:
+        - from:
+          to: 'GitOperationTask'

f8a_worker/dispatcher/nodes.yml

@@ -150,32 +150,17 @@
       import: 'f8a_worker.workers'
       max_retry: 0
       queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_BookkeeperTask_v0'
-    - name: 'package_keywords_tagging'
-      classname: 'PackageKeywordsTaggingTask'
-      storage_task_name: 'keywords_tagging'
+    - name: 'GitOperationTask'
+      classname: 'GitOperationTask'
       import: 'f8a_worker.workers'
       max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_package_keywords_tagging_v0'
-      storage: 'PackagePostgres'
-      selective_run_function:
-        name: 'selective_run_function'
-        import: 'f8a_worker.dispatcher.selective'
-    - name: 'keywords_tagging'
-      classname: 'KeywordsTaggingTask'
-      import: 'f8a_worker.workers'
-      max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_keywords_tagging_v0'
+      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_GitOperationTask_v0'
       storage: 'BayesianPostgres'
     - name: 'RepositoryDescCollectorTask'
       import: 'f8a_worker.workers'
       max_retry: 0
       queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_RepositoryDescCollectorTask_v0'
       storage: 'S3RepositoryDescription'
-    - name: 'KeywordsSummaryTask'
-      import: 'f8a_worker.workers'
-      max_retry: 0
-      queue: '{DEPLOYMENT_PREFIX}_{WORKER_ADMINISTRATION_REGION}_KeywordsSummaryTask_tagging_v0'
-      storage: 'S3KeywordsSummary'
     - name: 'GithubDependencyTreeTask'
       classname: 'GithubDependencyTreeTask'
       import: 'f8a_worker.workers'
@@ -218,13 +203,10 @@
     - 'bayesianApiPackageFlow'
     - 'bayesianPriorityPackageFlow'
     - 'bayesianPackageAnalysisFlow'
-    - 'bayesianPackageTaggingFlow'
     - 'componentApiFlow'
-    - 'keywordsSummaryFlow'
-    - 'keywordsApiSummaryFlow'
-    - 'keywordsPrioritySummaryFlow'
     - 'dependencyIngestFlow'
     - 'osioUserNotificationFlow'
+    - 'gitOperationsFlow'
 
   storages:
     - name: 'BayesianPostgres'
@@ -329,15 +311,6 @@
         encryption: false
         versioned: true
 
-    - name: 'S3KeywordsSummary'
-      import: 'f8a_worker.storages'
-      configuration:
-        <<: *configurationS3
-        bucket_name: '{DEPLOYMENT_PREFIX}-keywords-summary'
-        region_name: 'us-east-1'
-        encryption: false
-        versioned: true
-
     - name: 'S3UserIntent'
       import: 'f8a_worker.storages'
       configuration:

f8a_worker/dispatcher/selective.py

@@ -17,8 +17,7 @@ def selective_run_function(flow_name, node_name, node_args, task_names, storage_
     :return: ID of task that should be reused, None if task should be run again
     """
     try:
-        if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow',
-                         'bayesianPackageTaggingFlow'):
+        if flow_name in ('bayesianPackageFlow', 'bayesianPackageAnalysisFlow'):
             task_result = storage_pool.get_connected_storage('PackagePostgres').\
                 get_latest_task_entry(
                     node_args['ecosystem'],

f8a_worker/storages/__init__.py

@@ -15,7 +15,6 @@
 from .s3_gh_manifests import S3GitHubManifestMetadata
 from .s3_userprofilestore import S3UserProfileStore
 from .s3_description_repository import S3RepositoryDescription
-from .s3_keywords_summary import S3KeywordsSummary
 from .s3_userintent import S3UserIntent
 from .s3_manual_tagging import S3ManualTagging
 from .s3_crowd_source_tags import S3CrowdSourceTags
@@ -34,7 +33,6 @@
 assert S3GitHubManifestMetadata is not None
 assert S3UserProfileStore is not None
 assert S3RepositoryDescription is not None
-assert S3KeywordsSummary is not None
 assert S3UserIntent is not None
 assert S3ManualTagging is not None
 assert S3CrowdSourceTags is not None

f8a_worker/workers/__init__.py

@@ -12,9 +12,6 @@
 from f8a_worker.workers.graphaggregator import GraphAggregatorTask
 from f8a_worker.workers.init_analysis_flow import InitAnalysisFlow
 from f8a_worker.workers.init_package_flow import InitPackageFlow
-from f8a_worker.workers.keywords_summary import KeywordsSummaryTask
-from f8a_worker.workers.keywords_tagging import KeywordsTaggingTask
-from f8a_worker.workers.keywords_tagging import PackageKeywordsTaggingTask
 from f8a_worker.workers.libraries_io import LibrariesIoTask
 from f8a_worker.workers.license import LicenseCheckTask
 from f8a_worker.workers.mercator import MercatorTask
@@ -24,6 +21,7 @@
 from f8a_worker.workers.unknown_dep_fetcher import UnknownDependencyFetcherTask
 from f8a_worker.workers.user_notifier import UserNotificationTask
 from f8a_worker.workers.repo_dependency_finder import RepoDependencyFinderTask
+from f8a_worker.workers.git_operations import GitOperationTask
 
 # avoid Vulture and Pyflakes warnings
 assert BookkeeperTask is not None
@@ -38,9 +36,6 @@
 assert GraphAggregatorTask is not None
 assert InitAnalysisFlow is not None
 assert InitPackageFlow is not None
-assert KeywordsSummaryTask is not None
-assert KeywordsTaggingTask is not None
-assert PackageKeywordsTaggingTask is not None
 assert LibrariesIoTask is not None
 assert LicenseCheckTask is not None
 assert MercatorTask is not None
@@ -50,3 +45,4 @@
 assert UnknownDependencyFetcherTask is not None
 assert UserNotificationTask is not None
 assert RepoDependencyFinderTask is not None
+assert GitOperationTask is not None