diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 4f10cedac..223ef83cb 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -14,10 +14,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Make Executable - working-directory: ${{github.workspace}}/tools - run: sudo chmod +x makefsdata - - name: Use Node.js uses: actions/setup-node@v3 with: diff --git a/tools/linux_compile_makefsdata.sh b/tools/linux_compile_makefsdata.sh deleted file mode 100755 index 352c43ef9..000000000 --- a/tools/linux_compile_makefsdata.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -# This compiles makefsdata for Linux statically linking against musl -# - Make sure that you have the required packages installed (for Ubuntu these are: musl and musl-tools) -# - Download the latest release ZIP from https://github.com/richgel999/miniz/releases and copy miniz.c and miniz.h to -# $PICO_SDK_PATH/lib/lwip/src/apps/http - -musl-gcc \ - $PICO_SDK_PATH/lib/lwip/src/apps/http/makefsdata/makefsdata.c \ - --no-pie -static \ - -o makefsdata \ - -I$PICO_SDK_PATH/lib/lwip/src/include \ - -I$PICO_SDK_PATH/lib/lwip/contrib/ports/unix/lib \ - -I$PICO_SDK_PATH/lib/lwip/contrib/ports/unix/port/include \ - -DMAKEFS_SUPPORT_DEFLATE=1 diff --git a/tools/makefsdata b/tools/makefsdata deleted file mode 100755 index 8bdcc58e7..000000000 Binary files a/tools/makefsdata and /dev/null differ diff --git a/tools/makefsdata.darwin b/tools/makefsdata.darwin deleted file mode 100755 index 84ac07ab0..000000000 Binary files a/tools/makefsdata.darwin and /dev/null differ diff --git a/tools/makefsdata.exe b/tools/makefsdata.exe deleted file mode 100644 index 1c4ab23e6..000000000 Binary files a/tools/makefsdata.exe and /dev/null differ diff --git a/www/makefsdata.js b/www/makefsdata.js index fadc10ca1..7b436683e 100644 --- a/www/makefsdata.js +++ b/www/makefsdata.js @@ -1,41 +1,261 @@ -import path from "path"; -import fs from "fs"; +import path from 'node:path'; +import fs from 'node:fs'; -import { execFile } from "child_process"; -import { fileURLToPath } from "url"; +import { fileURLToPath } from 'node:url'; + +import pako from 'pako'; const __filename = fileURLToPath(import.meta.url); -const root = path.dirname(__filename).replace(path.normalize("www"), ""); +const root = path.dirname(__filename).replace(path.normalize('www'), ''); const rootwww = path.dirname(__filename); +const buildPath = path.join(rootwww, 'build'); + +const fsdataPath = path.normalize(path.join(root, 'lib/httpd/fsdata.c')); + +// These are the same content types that are used by the original makefsdata +const contentTypes = new Map([ + ['html', 'text/html'], + ['htm', 'text/html'], + [ + 'shtml', + 'text/html\r\nExpires: Fri, 10 Apr 2008 14:00:00 GMT\r\nPragma: no-cache', + ], + [ + 'shtm', + 'text/html\r\nExpires: Fri, 10 Apr 2008 14:00:00 GMT\r\nPragma: no-cache', + ], + [ + 'ssi', + 'text/html\r\nExpires: Fri, 10 Apr 2008 14:00:00 GMT\r\nPragma: no-cache', + ], + ['gif', 'image/gif'], + ['png', 'image/png'], + ['jpg', 'image/jpeg'], + ['bmp', 'image/bmp'], + ['ico', 'image/x-icon'], + ['class', 'application/octet-stream'], + ['cls', 'application/octet-stream'], + ['js', 'application/javascript'], + ['ram', 'application/javascript'], + ['css', 'text/css'], + ['swf', 'application/x-shockwave-flash'], + ['xml', 'text/xml'], + ['xsl', 'text/xml'], + ['pdf', 'application/pdf'], + ['json', 'application/json'], +]); + +const defaultContentType = 'text/plain'; + +const shtmlExtensions = new Set(['shtml', 'shtm', 'ssi', 'xml', 'json']); + +const skipCompressionExtensions = new Set(['png', 'json']); + +const serverHeader = 'GP2040-CE'; + +const payloadAlignment = 4; +const hexBytesPerLine = 16; + +function getFiles(dir) { + let results = []; + const list = fs.readdirSync(dir, { withFileTypes: true }); + for (const file of list) { + file.path = dir + '/' + file.name; + if (file.isDirectory()) { + results = results.concat(getFiles(file.path)); + } else if (file.isFile()) { + results.push(file.path); + } + } + return results; +} + +function getLowerCaseFileExtension(file) { + const ext = file.split('.').pop(); + return ext ? ext.toLowerCase() : ''; +} + +function fixFilenameForC(filename) { + let varName = ''; + for (let i = 0; i < filename.length; i++) { + const c = filename[i]; + if (c >= 'a' && c <= 'z') { + varName += c; + } else if (c >= 'A' && c <= 'Z') { + varName += c; + } else if (c >= '0' && c <= '9') { + varName += c; + } else { + varName += '_'; + } + } + return varName; +} + +function CStringLength(str) { + return Buffer.from(str, 'utf8').length; +} + +function createHexString(value, prependComment = false) { + let column = 0; + let hexString = ''; + if (typeof value === 'string') { + const bytes = Buffer.from(value, 'utf8'); + if (prependComment) { + hexString += `/* "${value}" (${bytes.length} bytes) */\n`; + } + bytes.forEach((byte) => { + hexString += '0x' + byte.toString(16).padStart(2, '0') + ','; + if (++column >= hexBytesPerLine) { + hexString += '\n'; + column = 0; + } + }); + } else if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + value.forEach((byte) => { + hexString += '0x' + byte.toString(16).padStart(2, '0') + ','; + if (++column >= hexBytesPerLine) { + hexString += '\n'; + column = 0; + } + }); + } else { + throw new Error('Invalid value type'); + } -const fsdata_filename = path.normalize("/lib/httpd/fsdata.c"); - -function correctincludes() { - fs.readFile(root + fsdata_filename, "utf8", function (error, data) { - if (data) { - let result = data.replace("#include \"lwip/apps/fs.h\"\r\n#include \"lwip/def.h\"", '#include "fsdata.h"'); - - fs.writeFile(root + fsdata_filename, result, "utf8", function (error) { - if (error) { - console.error(error); - } - }); - } else { - if (error) { - console.error(error); - } - } - }); + return hexString + '\n'; } function makefsdata() { - execFile(path.normalize(process.platform !== "darwin" ? `${root}/tools/makefsdata` : `${root}/tools/makefsdata.darwin`), [path.normalize(`${rootwww}/build`), '-defl:10', '-xc:png,json', `-f:`+ path.normalize(`${root}/lib/httpd/fsdata.c`)], function(error, data) { - if (error) { - console.error(error); - } else { - correctincludes(); - } - }); + let fsdata = ''; + fsdata += '#include "fsdata.h"\n'; + fsdata += '\n'; + fsdata += '#define file_NULL (struct fsdata_file *) NULL\n'; + fsdata += '\n'; + fsdata += '#ifndef FS_FILE_FLAGS_HEADER_INCLUDED\n'; + fsdata += '#define FS_FILE_FLAGS_HEADER_INCLUDED 1\n'; + fsdata += '#endif\n'; + fsdata += '#ifndef FS_FILE_FLAGS_HEADER_PERSISTENT\n'; + fsdata += '#define FS_FILE_FLAGS_HEADER_PERSISTENT 0\n'; + fsdata += '#endif\n'; + fsdata += '/* FSDATA_FILE_ALIGNMENT: 0=off, 1=by variable, 2=by include */\n'; + fsdata += '#ifndef FSDATA_FILE_ALIGNMENT\n'; + fsdata += '#define FSDATA_FILE_ALIGNMENT 0\n'; + fsdata += '#endif\n'; + fsdata += '#ifndef FSDATA_ALIGN_PRE\n'; + fsdata += '#define FSDATA_ALIGN_PRE\n'; + fsdata += '#endif\n'; + fsdata += '#ifndef FSDATA_ALIGN_POST\n'; + fsdata += '#define FSDATA_ALIGN_POST\n'; + fsdata += '#endif\n'; + fsdata += '#if FSDATA_FILE_ALIGNMENT==2\n'; + fsdata += '#include "fsdata_alignment.h"\n'; + fsdata += '#endif\n'; + fsdata += '\n'; + + let payloadAlignmentDummyCounter = 0; + + const fileInfos = []; + + getFiles(buildPath).forEach((file) => { + const ext = getLowerCaseFileExtension(file); + + const qualifiedName = '/' + path.relative(buildPath, file).replace(/\\/g, '/'); + const varName = fixFilenameForC(qualifiedName); + + fsdata += '#if FSDATA_FILE_ALIGNMENT==1\n'; + fsdata += `static const unsigned int dummy_align_${varName} = ${payloadAlignmentDummyCounter++};\n`; + fsdata += '#endif\n'; + + const fileContent = fs.readFileSync(file); + let compressed = fileContent.buffer; + let isCompressed = false; + if (!skipCompressionExtensions.has(ext)) { + compressed = pako.deflate(fileContent, { + level: 9, + windowBits: 15, + memLevel: 9, + }); + console.log( + `Compressed ${qualifiedName} from ${fileContent.byteLength} to ${ + compressed.byteLength + } bytes (${Math.round( + (compressed.byteLength / fileContent.byteLength) * 100, + )}%)`, + ); + if (compressed.byteLength >= fileContent.byteLength) { + console.log( + `Skipping compression of ${qualifiedName}, compressed size is larger than original`, + ); + } else { + isCompressed = true; + } + } else { + console.log(`Skipping compression of ${qualifiedName} by file extension`); + } + + const qualifiedNameLength = CStringLength(qualifiedName) + 1; + const paddedQualifiedNameLength = + Math.ceil(qualifiedNameLength / payloadAlignment) * payloadAlignment; + // Zero terminate the qualified name and pad it to the next multiple of payloadAlignment + const paddedQualifiedName = + qualifiedName + + '\0'.repeat(1 + paddedQualifiedNameLength - qualifiedNameLength); + fsdata += `static const unsigned char data_${varName}[] FSDATA_ALIGN_PRE = {\n`; + fsdata += `/* ${qualifiedName} (${qualifiedNameLength} chars) */\n`; + fsdata += createHexString(paddedQualifiedName, false); + fsdata += '\n'; + fsdata += '/* HTTP header */\n'; + fsdata += createHexString('HTTP/1.0 200 OK\r\n', true); + fsdata += createHexString(`Server: ${serverHeader}\r\n`, true); + fsdata += createHexString( + `Content-Length: ${ + isCompressed ? compressed.byteLength : fileContent.byteLength + }\r\n`, + true, + ); + if (isCompressed) { + fsdata += createHexString('Content-Encoding: deflate\r\n', true); + } + fsdata += createHexString( + `Content-Type: ${contentTypes.get(ext) ?? defaultContentType}\r\n\r\n`, + true, + ); + fsdata += `/* raw file data (${ + isCompressed ? compressed.byteLength : fileContent.byteLength + } bytes) */\n`; + fsdata += createHexString(isCompressed ? compressed : fileContent); + fsdata += '};\n\n'; + + fileInfos.push({ + varName, + paddedQualifiedNameLength, + isSsiFile: shtmlExtensions.has(ext), + }); + }); + + let prevFile = 'NULL'; + fileInfos.forEach((fileInfo) => { + fsdata += `const struct fsdata_file file_${fileInfo.varName}[] = {{\n`; + fsdata += `file_${prevFile},\n`; + fsdata += `data_${fileInfo.varName},\n`; + fsdata += `data_${fileInfo.varName} + ${fileInfo.paddedQualifiedNameLength},\n`; + fsdata += `sizeof(data_${fileInfo.varName}) - ${fileInfo.paddedQualifiedNameLength},\n`; + fsdata += `FS_FILE_FLAGS_HEADER_INCLUDED | ${ + fileInfo.isSsiFile + ? 'FS_FILE_FLAGS_SSI' + : 'FS_FILE_FLAGS_HEADER_PERSISTENT' + }\n`; + fsdata += '}};\n\n'; + + prevFile = fileInfo.varName; + }); + + fsdata += `#define FS_ROOT file_${prevFile}\n`; + fsdata += `#define FS_NUMFILES ${fileInfos.length}\n`; + + fs.writeFileSync(fsdataPath, fsdata, 'utf8'); } -makefsdata(); \ No newline at end of file + +makefsdata(); diff --git a/www/package-lock.json b/www/package-lock.json index a8f549bac..30d044bbe 100644 --- a/www/package-lock.json +++ b/www/package-lock.json @@ -40,6 +40,7 @@ "eslint-plugin-react-refresh": "^0.4.1", "express": "^4.18.2", "nodemon": "^2.0.22", + "pako": "^2.1.0", "prettier": "^3.0.1", "sass": "^1.62.1", "source-map-explorer": "^2.5.3", @@ -4282,6 +4283,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", + "integrity": "sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==", + "dev": true + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/www/package.json b/www/package.json index dd1b851f7..b93e6b101 100644 --- a/www/package.json +++ b/www/package.json @@ -47,6 +47,7 @@ "eslint-plugin-react-refresh": "^0.4.1", "express": "^4.18.2", "nodemon": "^2.0.22", + "pako": "^2.1.0", "prettier": "^3.0.1", "sass": "^1.62.1", "source-map-explorer": "^2.5.3",