Merge pull request #44 from alkem-io/release/v0.8.0

Release version 0.8.0

.env.default

@@ -23,3 +23,5 @@ VECTOR_DB_PORT=8000
 
 CHUNK_SIZE=1000
 CHUNK_OVERLAP=100
+
+BATCH_SIZE=20

graphql/fragments/profile.graphql

@@ -0,0 +1,21 @@
+fragment ProfileFields on Profile {
+  id
+  description
+  displayName
+  tagline
+  url
+  type
+  tagset {
+    tags
+  }
+  references {
+    description
+    name
+    uri
+  }
+  visuals {
+    uri
+    name
+  }
+}
+

graphql/fragments/space-ingest.graphql

@@ -36,6 +36,7 @@ fragment SpaceIngest on Space {
       id
       nameID
       type
+      visibility
       comments {
         messagesCount
         messages {
@@ -58,43 +59,26 @@ fragment SpaceIngest on Space {
         }
       }
       framing {
+        id
         profile {
-          description
-          displayName
-          tagline
-          url
-          tagset {
-            tags
-          }
-          references {
-            description
-            name
-            uri
-          }
-          visuals {
-            uri
-            name
-          }
+          ...ProfileFields
         }
       }
       contributions {
+        post {
+          id
+          nameID
+          profile {
+            ...ProfileFields
+          }
+        }
         link {
+          id
           uri
           profile {
-            description
-            displayName
-            url
-            type
-            references {
-              description
-              name
-              uri
-            }
-            visuals {
-              uri
-              name
-            }
+            ...ProfileFields
           }
+
         }
       }
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alkemio/space-ingest",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "description": "",
   "author": "Alkemio Foundation",
   "private": true,

src/callout.handlers/base.ts

@@ -1,29 +1,16 @@
-import { Callout } from '../generated/graphql';
+import { Callout, CalloutContribution } from '../generated/graphql';
 import { Document } from 'langchain/document';
 import generateDocument from '../generate.document';
 
 export const baseHandler = async (
   callout: Partial<Callout>
 ): Promise<Document[]> => {
   const { id: documentId, type } = callout;
+
   const generated = generateDocument(callout.framing);
   const { title, source } = generated;
   let pageContent = generated.pageContent;
 
-  // extra loop but will do for now
-  const contributions = callout.contributions
-    ?.filter((article: any) => !!article.link)
-    .map((contribution: any) => {
-      const { pageContent: contribArticle } = generateDocument(
-        contribution.link
-      );
-      return contribArticle;
-    })
-    .join('\n');
-
-  if (contributions)
-    pageContent = `${pageContent}\nContributions:\n${contributions}`;
-
   const messages = callout.comments?.messages || [];
   const processedMessages: string[] = [];
   for (let i = 0; i < messages.length; i++) {
@@ -44,7 +31,7 @@ export const baseHandler = async (
   if (processedMessages.length)
     pageContent = `${pageContent}\nMessages:\n${processedMessages.join('\n')}`;
 
-  return [
+  const result: Document[] = [
     new Document({
       pageContent,
       metadata: {
@@ -55,4 +42,31 @@ export const baseHandler = async (
       },
     }),
   ];
+
+  // extra loop but will do for now
+  callout.contributions
+    ?.map((contribution: Partial<CalloutContribution>) => {
+      let docLike;
+      if (!!contribution.link) {
+        docLike = contribution.link;
+      } else if (!!contribution.post) {
+        docLike = contribution.post;
+      }
+      const { pageContent, documentId, source, type, title } =
+        generateDocument(docLike);
+      result.push(
+        new Document({
+          pageContent,
+          metadata: {
+            documentId,
+            source,
+            type,
+            title,
+          },
+        })
+      );
+    })
+    .join('\n');
+
+  return result;
 };

src/constants.ts

@@ -0,0 +1,3 @@
+export const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE || '1000');
+export const CHUNK_OVERLAP = parseInt(process.env.CHUNK_OVERLAP || '100');
+export const BATCH_SIZE = parseInt(process.env.BATCH_SIZE || '20');

src/generate.document.ts

@@ -1,3 +1,4 @@
+import { Reference, Visual } from './generated/graphql';
 import { DocumentType, mapType } from './document.type';
 
 interface GeneratedDocument {
@@ -8,6 +9,7 @@ interface GeneratedDocument {
   title: string;
 }
 
+//TODO type this pls
 export default (docLike: any): GeneratedDocument => {
   const {
     id: documentId,
@@ -21,9 +23,11 @@ export default (docLike: any): GeneratedDocument => {
       displayName,
       location,
       visuals,
+      type: profileType,
     },
     context,
   } = docLike;
+
   const { vision, impact, who } = context || {};
   const { city, country, postalCode } = location || {};
 
@@ -37,7 +41,7 @@ export default (docLike: any): GeneratedDocument => {
   if (who) pageContent = `${pageContent}\nWho: ${who}`;
 
   const processedVisuals = visuals
-    .map((visual: any) => `\t${visual.name}: ${visual.uri}`)
+    .map((visual: Visual) => `\t${visual.name}: ${visual.uri}`)
     .join('\n');
   if (processedVisuals)
     pageContent = `${pageContent}\nVisuals: ${processedVisuals}`;
@@ -46,7 +50,7 @@ export default (docLike: any): GeneratedDocument => {
     pageContent = `${pageContent}\nLocation: ${postalCode} ${city} ${country}`;
 
   const processedRefs = references.map(
-    ({ description, name, uri }: any) =>
+    ({ description, name, uri }: Reference) =>
       `\tReference name: ${name}\n\tReference description: ${description}\n\tUri: ${uri}\n`
   );
   if (processedRefs)
@@ -56,7 +60,7 @@ export default (docLike: any): GeneratedDocument => {
   return {
     documentId,
     source,
-    type: mapType(type),
+    type: mapType(type ?? profileType),
     pageContent,
     title: displayName,
   };