feat(dash): Project-level dashboards, vizes, and calculation topsheets

Create a folder named 'simwrapper' at any level relative to your current
output folder (inclusive). All folders will inherit the YAML files from
those folders.

File paths in YAML are always interpret as if they are in the
currently viewed folder, NOT the folder from which the YAML file is stored.


Squashed commit of the following:

commit d5cef22778b18b688e69ea53ea213c3172840eee
Author: Billy Charlton <charlton@vsp.tu-berlin.de>
Date:   Thu Dec 16 11:13:15 2021 +0100

    feat(dash): viz-* yaml files can now be in any simwrapper/ folder

    Just like dashboards and topsheets!

commit ea428fe4f9eabca0a7ea1830ec6b7ba91e61b51e
Author: Billy Charlton <charlton@vsp.tu-berlin.de>
Date:   Wed Dec 15 18:37:33 2021 +0100

    Dashboards and topsheets can be in any nearby simwrapper/ folder

    Dashboard and topsheet YAML configurations can now be in the current
    folder OR in any simwrapper folder at any level in the tree hierarchy
    at this level or above.

    In other words, if you are looking at ~/data/berlin/myrun01,
    dashboards and topsheets will be found in that folder and also in:

    ~/simwrapper
    ~/data/simwrapper
    ~/data/berlin/simwrapper
    ~/data/berlin/myrun01/simwrapper

    Configuration files of the same name get superceded; the closest
    to the current folder take precedence; thus a project can have defaults
    but you can override them if you like.

commit 63308fb4237a2d829305172b241c1e04ede63894
Author: Billy Charlton <charlton@vsp.tu-berlin.de>
Date:   Wed Dec 15 13:28:19 2021 +0100

    cache & optimize directory fetching

src/Globals.ts

@@ -15,6 +15,12 @@ export enum Status {
   WARNING,
 }
 
+export type YamlConfigs = {
+  dashboards: { [filename: string]: string }
+  topsheets: { [filename: string]: string }
+  vizes: { [filename: string]: string }
+}
+
 export const UI_FONT =
   "'Titillium Web', 'Roboto', 'Open Sans', Avenir, Arial, Helvetica, sans-serif"
 

src/components/TopSheet/TopSheet.vue

@@ -15,7 +15,7 @@
 </template>
 
 <script lang="ts">
-import { FileSystemConfig } from '@/Globals'
+import { FileSystemConfig, YamlConfigs } from '@/Globals'
 import { Vue, Component, Watch, Prop } from 'vue-property-decorator'
 
 import TopSheetWorker from './TopSheetWorker.worker.ts?worker'
@@ -35,11 +35,13 @@ export default class VueComponent extends Vue {
   @Prop({ required: true })
   private subfolder!: string
 
-  @Prop({ required: true })
-  private files!: string[]
+  @Prop({ required: true }) private files!: string[]
 
-  @Prop({ required: true })
-  private yaml!: string
+  @Prop({ required: true }) private yaml!: string
+
+  @Prop({ required: true }) private allConfigFiles!: YamlConfigs
+
+  @Prop({ required: false }) private cardId?: string
 
   private solverThread!: any
 
@@ -112,6 +114,7 @@ export default class VueComponent extends Vue {
         files: this.files,
         yaml: this.yaml,
         locale: this.$store.state.locale,
+        allConfigFiles: this.allConfigFiles,
       })
     } catch (e) {
       const message = '' + e
@@ -126,7 +129,8 @@ export default class VueComponent extends Vue {
     const data = message.data
     switch (data.response) {
       case 'title':
-        this.title = data.title
+        if (this.cardId) this.$emit('titles', data.title)
+        else this.title = data.title
         break
       case 'entries':
         this.entries = data.entryFields
@@ -146,12 +150,13 @@ export default class VueComponent extends Vue {
 @import '@/styles.scss';
 
 h3.curate-heading {
-  font-size: 1.8rem;
+  font-size: 1.6rem;
   font-weight: bold;
   color: var(--textFancy);
   padding-top: 0.5rem;
   margin-top: 0rem;
   margin-bottom: 0.5rem;
+  border-bottom: 1px dotted var(--textFancy);
 }
 
 .curate-content {

src/components/TopSheet/TopSheetWorker.worker.ts

@@ -5,7 +5,7 @@ import pako from 'pako'
 import Papaparse from 'papaparse'
 import YAML from 'yaml'
 
-import { FileSystemConfig } from '@/Globals'
+import { FileSystemConfig, YamlConfigs } from '@/Globals'
 import HTTPFileSystem from '@/js/HTTPFileSystem'
 import { findMatchingGlobInFiles, parseXML } from '@/js/util'
 import globalStore from '@/store'
@@ -59,6 +59,12 @@ let _calculations: any = {}
 let _yamlFile: string = ''
 let _locale = 'en'
 
+let _allConfigYamls: YamlConfigs = {
+  dashboards: {},
+  topsheets: {},
+  vizes: {},
+}
+
 const _fileData: any = {}
 
 const testRows: TableRow[] = [
@@ -116,6 +122,7 @@ async function runTopSheet(props: {
   files: string[]
   yaml: string
   locale: string
+  allConfigFiles: YamlConfigs
 }) {
   // console.log('TopSheet thread worker starting')
 
@@ -124,6 +131,7 @@ async function runTopSheet(props: {
   _files = props.files
   _yamlFile = props.yaml
   _locale = props.locale
+  _allConfigYamls = props.allConfigFiles
 
   // read the table definitions from yaml
   _yaml = await getYaml()
@@ -308,7 +316,12 @@ function getFileVariableReplacements(expr: string) {
 }
 
 async function getYaml() {
-  const text = await _fileSystem.getFileText(_subfolder + '/' + _yamlFile)
+  let filename = _yamlFile
+
+  // if we have a reference to a yaml in a different folder, use that one
+  filename = _allConfigYamls.topsheets[_yamlFile] || filename
+
+  const text = await _fileSystem.getFileText(filename)
   const yaml = YAML.parse(text) as TopsheetYaml
   return yaml
 }

src/components/TopsheetsFinder/TopsheetsFinder.vue

@@ -5,6 +5,7 @@
     :subfolder="subfolder"
     :files="files"
     :yaml="sheet"
+    :allConfigFiles="allConfigFiles"
   )
 
 </template>
@@ -16,9 +17,7 @@ const i18n = {
   },
 }
 
-import micromatch from 'micromatch'
-
-import { FileSystemConfig } from '@/Globals'
+import { FileSystemConfig, YamlConfigs } from '@/Globals'
 import { Vue, Component, Watch, Prop } from 'vue-property-decorator'
 import HTTPFileSystem from '@/js/HTTPFileSystem'
 import TopSheet from '@/components/TopSheet/TopSheet.vue'
@@ -34,65 +33,24 @@ export default class VueComponent extends Vue {
   @Prop({ required: true })
   private files!: string[]
 
-  private topsheets: any[] = []
-
   private fileSystem!: HTTPFileSystem
+  private allConfigFiles!: YamlConfigs
+  private topsheets: any[] = []
 
-  @Watch('subfolder') folderUpdated() {
-    this.topsheets = []
-  }
-
-  @Watch('files') async filesUpdated() {
+  @Watch('subfolder')
+  @Watch('files')
+  private async filesUpdated() {
     if (this.files.length) {
       this.fileSystem = new HTTPFileSystem(this.fileSystemConfig)
       this.topsheets = await this.findTopsheetsForThisFolder()
-      // console.log('TOPSHEETS', this.topsheets)
     }
   }
 
   private async mounted() {}
 
-  private async findTopsheetsForThisFolder() {
-    const folders = this.findAllTopsheetFolders()
-    const topsheets = {} as any
-
-    // get list of all topsheet.yaml files, overwriting/overriding as we drill down
-    for (const folder of folders) {
-      const { files } = await this.fileSystem.getDirectory(folder)
-      for (const file of files) topsheets[file] = `${folder}/${file}`
-    }
-
-    // use local files in this folder instead of any we just found
-    const matches = micromatch(this.files, ['topsheet*.y?(a)ml'])
-    for (const match of matches) topsheets[match] = `${match}`
-
-    return Object.values(topsheets)
-  }
-
-  private findAllTopsheetFolders() {
-    // Check for any .topsheets folders, here or above
-    // BUT override topsheets with topsheets in this folder
-
-    const allFolders = localStorage.getItem('RunFinder.foundFolders')
-    if (!allFolders) return []
-
-    const topsheetEntries = JSON.parse(allFolders)[this.fileSystemConfig.name].filter((f: any) => {
-      return f.path.endsWith('.topsheets')
-    })
-    const topsheetFolders = topsheetEntries.map((f: any) => f.path)
-
-    // We only want the ones that are at this point in the tree or above us.
-    const relevantTopsheetFolders = topsheetFolders.filter((f: string) => {
-      // ----> i am in: /data/berlin
-      // /data/berlin/.topsheets  -> true
-      // /data/.topsheets -> true
-      // /data/emissions/.topsheets -> false
-      // ----> so remove "/.topsheets" and see if our folder begins with that
-      const candidate = f.substring(0, f.length - 11)
-      return this.subfolder.startsWith(candidate)
-    })
-
-    return relevantTopsheetFolders
+  private async findTopsheetsForThisFolder(): Promise<string[]> {
+    this.allConfigFiles = await this.fileSystem.findAllYamlConfigs(this.subfolder)
+    return Object.values(this.allConfigFiles.topsheets)
   }
 }
 </script>

src/js/HTTPFileSystem.ts

@@ -1,6 +1,14 @@
-import { DirectoryEntry, FileSystemConfig } from '@/Globals'
+import micromatch from 'micromatch'
+import { DirectoryEntry, FileSystemConfig, YamlConfigs } from '@/Globals'
 import globalStore from '@/store'
 
+const YAML_FOLDER = 'simwrapper'
+
+// Cache directory listings for each slug & directory
+const CACHE: { [slug: string]: { [dir: string]: DirectoryEntry } } = {}
+
+// ---------------------------------------------------------------------------
+
 class SVNFileSystem {
   private baseUrl: string
   private urlId: string
@@ -12,6 +20,12 @@ class SVNFileSystem {
 
     this.baseUrl = project.baseURL
     if (!project.baseURL.endsWith('/')) this.baseUrl += '/'
+
+    if (!CACHE[this.urlId]) CACHE[this.urlId] = {}
+  }
+
+  public clearCache() {
+    CACHE[this.urlId] = {}
   }
 
   public cleanURL(scaryPath: string) {
@@ -38,7 +52,7 @@ class SVNFileSystem {
     }
 
     const myRequest = new Request(path, { headers })
-    const response = await fetch(myRequest).then(response => {
+    const response = await fetch(myRequest).then((response) => {
       // Check HTTP Response code: 200 is OK, everything else is a problem
       if (response.status != 200) {
         console.log('Status:', response.status)
@@ -82,10 +96,78 @@ class SVNFileSystem {
     // don't download any files!
     if (!stillScaryPath.endsWith('/')) stillScaryPath += '/'
 
+    // Use cached version if we have it
+    const cachedEntry = CACHE[this.urlId][stillScaryPath]
+    if (cachedEntry) return cachedEntry
+
+    // Generate and cache the listing
     const response = await this._getFileResponse(stillScaryPath).then()
     const htmlListing = await response.text()
+    const dirEntry = this.buildListFromHtml(htmlListing)
+    CACHE[this.urlId][stillScaryPath] = dirEntry
+
+    return dirEntry
+  }
+
+  async findAllYamlConfigs(folder: string): Promise<YamlConfigs> {
+    const yamls: YamlConfigs = { dashboards: {}, topsheets: {}, vizes: {} }
+
+    const configFolders = []
+
+    // first find all simwrapper folders
+    let currentPath = '/'
+    const { dirs } = await this.getDirectory(currentPath)
+    if (dirs.indexOf(YAML_FOLDER) > -1)
+      configFolders.push(`${currentPath}/${YAML_FOLDER}`.replaceAll('//', '/'))
+
+    const pathChunks = folder.split('/')
+    for (const chunk of pathChunks) {
+      currentPath = `${currentPath}${chunk}/`
+      const { dirs } = await this.getDirectory(currentPath)
+      if (dirs.indexOf(YAML_FOLDER) > -1)
+        configFolders.push(`${currentPath}/${YAML_FOLDER}`.replaceAll('//', '/'))
+    }
+
+    // also add current working folder as final option, which supercedes all others
+    configFolders.push(folder)
+
+    // console.log('configFolders', configFolders)
+
+    // find all dashboards, topsheets, and viz-* yamls in each configuration folder.
+    // Overwrite keys as we go; identically-named configs from parent folders get superceded as we go.
+    const dashboard = 'dashboard*.y?(a)ml'
+    const topsheet = 'topsheet*.y?(a)ml'
+    const viz = 'viz*.y?(a)ml'
+
+    for (const configFolder of configFolders) {
+      const { files } = await this.getDirectory(configFolder)
+
+      micromatch
+        .match(files, dashboard)
+        .map((yaml) => (yamls.dashboards[yaml] = `${configFolder}/${yaml}`.replaceAll('//', '/')))
+
+      micromatch
+        .match(files, topsheet)
+        .map((yaml) => (yamls.topsheets[yaml] = `${configFolder}/${yaml}`.replaceAll('//', '/')))
+
+      micromatch
+        .match(files, viz)
+        .map((yaml) => (yamls.vizes[yaml] = `${configFolder}/${yaml}`.replaceAll('//', '/')))
+    }
 
-    return this.buildListFromHtml(htmlListing)
+    // Sort them all by filename
+    yamls.dashboards = Object.fromEntries(
+      Object.entries(yamls.dashboards).sort((a, b) => (a[0] > b[0] ? 1 : -1))
+    )
+    yamls.topsheets = Object.fromEntries(
+      Object.entries(yamls.topsheets).sort((a, b) => (a[0] > b[0] ? 1 : -1))
+    )
+    yamls.vizes = Object.fromEntries(
+      Object.entries(yamls.vizes).sort((a, b) => (a[0] > b[0] ? 1 : -1))
+    )
+
+    console.log(yamls)
+    return yamls
   }
 
   private buildListFromHtml(data: string): DirectoryEntry {

src/js/RunFinder.ts

@@ -5,7 +5,7 @@ import { FileSystemConfig } from '@/Globals'
 
 let foundFolders = { number: 0, folders: {} as any }
 
-const findRuns = function() {
+const findRuns = function () {
   // check date and time
   const storedUpdate = localStorage.getItem('RunFinder.lastUpdate')
   const lastUpdate = storedUpdate ? parseInt(storedUpdate) : 0
@@ -38,37 +38,34 @@ const populate = () => {
   foundFolders = { number: 0, folders: {} }
   store.commit('updateRunFolders', foundFolders)
 
-  store.state.svnProjects.forEach(root => {
+  store.state.svnProjects.forEach((root) => {
     if (!root.hidden && root.slug !== 'gallery') drillIntoRootProject(root)
   })
 }
 
-const drillIntoRootProject = function(root: FileSystemConfig) {
+const drillIntoRootProject = function (root: FileSystemConfig) {
   console.log('Drilling into:', root.name)
+
   const fileSystem = new HTTPFileSystem(root)
+  fileSystem.clearCache()
 
   foundFolders.folders[root.name] = []
 
   fetchFolders(root, fileSystem, '')
 }
 
-const fetchFolders = async function(
+const fetchFolders = async function (
   root: FileSystemConfig,
   fileSystem: HTTPFileSystem,
   folder: string
 ) {
   try {
     // skip some big folders we know we don't care about
-    if (root.skipList && root.skipList.filter(f => folder.endsWith(f)).length) return
+    if (root.skipList && root.skipList.filter((f) => folder.endsWith(f)).length) return
 
     // skip .dot and __MACOSX folders
     if (folder.endsWith('__MACOSX')) return
-    if (
-      folder
-        .split('/')
-        .pop()
-        ?.startsWith('.')
-    ) {
+    if (folder.split('/').pop()?.startsWith('.')) {
       return
     }