Extend image embedding on Unix with support for gzipped compressed im…

…ages

(64-bit images are compressed to about 1/7.5 times their original size).
Extend the image embedding preprocessor (genUnixImageResource.c) to run
gzip if given a -z argument.

deploy/packaging/genUnixImageResource.c

@@ -1,51 +1,79 @@
-// Take as input an image file. Produce as output a C file defining
-// a byte array called embeddedImage which contains the contents of
-// the image file, and a string called embeddedImageName containing
-// the base name of the image.
+// Take as input an image file. Produce as output a C file defining a byte
+// array called embeddedImage which contains the contents of the image
+// file, a string called embeddedImageName containing the base name of the
+// image, and an unsigned long, called embeddedImageSize, containing the
+// uncompressed size of the image. If the -z flag is supplied, compress the
+// image via gzip and include the compressed data, not the original.
 
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <limits.h>
+#include <unistd.h>
 
 #define NITEMS (80/5)
 int
 main(int argc, char *argv[])
 {
-    FILE *ifh, *ofh;
+    FILE *ifh, *ofh, *ttyfh;
 	unsigned char data[NITEMS];
+	char compressCommandOrFile[PATH_MAX+16];
 	char *basename, *dot;
 	size_t n;
+	int compress = argc > 1 && !strcmp(argv[1],"-z");
 
-	if (argc == 2 || argc == 3) {
-		if (!(ifh = fopen(argv[1], "rb"))) {
-			perror("fopen");
-			return 1;
+	if (argc != 2 + compress && argc != 3 + compress) {
+		printf("usage: %s [-z] file.image [output file]\n", argv[0]);
+		return 1;
+	}
+	if (compress) {
+		char *mytty = ttyname(fileno(stdin));
+		FILE *ttyfh;
+
+		if (!(ttyfh = fopen(mytty,"w+"))) {
+			fprintf(stderr,"failed to open %s\n", mytty);
+			return 2;
 		}
-		if (argc == 3) {
-			if (!(ofh = fopen(argv[2], "w"))) {
-				perror("fopen");
-				return 2;
-			}
+		sprintf(compressCommandOrFile,"gzip -9 -k %s\n",argv[2]);//-9=best compression,-k=keep input
+		fprintf(ttyfh,"running gzip to compress file %s...\n",argv[2]);
+		if (system(compressCommandOrFile)) {
+			fprintf(stderr,"failed to run gzip to compress file\n");
+			return 3;
 		}
-		else
-			ofh = stdout;
 	}
-	else {
-		printf("usage: %s file.image [output file]\n", argv[0]);
-		exit(2);
+	if (!(ifh = fopen(argv[1+compress], "rb"))) {
+		perror("fopen input file");
+		return 4;
+	}
+	if (argc == 2 + compress)
+		ofh = stdout;
+	else if (!(ofh = fopen(argv[2+compress], "w"))) {
+		perror("fopen output file");
+		return 5;
 	}
-	if ((basename = strrchr(argv[1],'/')))
+	if ((basename = strrchr(argv[1+compress],'/')))
 		basename = basename + 1;
 	else
-		basename = argv[1];
+		basename = argv[1+compress];
 	if (!(dot = strrchr(basename,'.')))
-		dot = basename + strlen(basename) - 1;
+		dot = basename + strlen(basename);
 	fprintf(ofh,
 			"char embeddedImageName[] = \"%.*s\";\n",
 			(int)(dot - basename), basename);
 	fseek(ifh,0,SEEK_END);
 	fprintf(ofh,"unsigned long embeddedImageSize = %ld;\n",ftell(ifh));
 	fseek(ifh,0,SEEK_SET);
+	if (compress) {
+		(void)fclose(ifh);
+		sprintf(compressCommandOrFile,"%s.gz",argv[2]);
+		if (!(ifh = fopen(compressCommandOrFile, "rb"))) {
+			perror("fopen compressed file");
+			return 6;
+		}
+		fseek(ifh,0,SEEK_END);
+		fprintf(ofh,"unsigned long embeddedCompressedDataSize = %ld;\n",ftell(ifh));
+		fseek(ifh,0,SEEK_SET);
+	}
 	fprintf(ofh, "unsigned char embeddedImage[] = {\n");
 	while ((n = fread(data,sizeof(unsigned char),NITEMS,ifh)) > 0) {
 		unsigned char lastchar = feof(ifh) ? '\n' : ',';

platforms/unix/vm/sqImageFileAccess.h

@@ -14,13 +14,22 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
+#if INCLUDE_SIF_CODE
+# include <zlib.h>
+#endif
 
-// On Unix we use the native file interface. There is also support for embedded images
+// On Unix we use the native file interface. There is also support for embedded images,
+// which may be compressed with gzip. See deploy/packaging/genUnixImageResource.c
 
 #define sqImageFile	int
 #define invalidSqImageFile(sif) ((sif) < 0)
 #define squeakFileOffsetType off_t
 #define ImageIsEmbedded ((sqImageFile)1)
+#define ImageIsEmbeddedAndCompressed ((sqImageFile)2)
+
+#define GZIPMagic0 0x1f
+#define GZIPMagic1 0x8b
+
 
 // Save/restore.
 
@@ -49,7 +58,14 @@ void sqImageFileSeekEnd(sqImageFile f,off_t pos);
 
 #if INCLUDE_SIF_CODE
 
-static int sIFOMode;
+static int sIFOMode; // input/output file mode
+
+// Support for image embedded as a resource
+static unsigned char *eiData = NULL;
+static unsigned long eiSize = 0, eicSize = 0;
+static off_t eiReadPosition = 0;
+static z_stream eizs = { 0, }; // embedded image zlib stream
+
 
 sqImageFile
 sqImageFileOpen(const char *fileName, const char *mode)
@@ -71,6 +87,10 @@ extern sqInt failed(void);
 
 	if (f == ImageIsEmbedded)
 		return;
+	if (f == ImageIsEmbeddedAndCompressed) {
+		(void)inflateEnd(&eizs);
+		return;
+	}
 
 	if (!failed()
 	 && sIFOMode == O_RDWR+O_CREAT
@@ -81,16 +101,12 @@ extern sqInt failed(void);
 		perror("sqImageFileClose close");
 }
 
-// Support for image embedded as a resource
-static unsigned char *eiData = NULL;
-static unsigned long eiSize = 0;
-static off_t eiReadPosition = 0;
-
 static inline void
-noteEmbeddedImage(unsigned char *data, unsigned long size)
+noteEmbeddedImage(unsigned char *data, unsigned long size, unsigned long csize)
 {
 	eiData = data;
 	eiSize = size;
+	eicSize = csize;
 }
 
 int
@@ -110,6 +126,69 @@ sqEmbeddedImageRead(void *ptr, size_t sz, size_t count)
 }
 
 
+static inline size_t
+sqCompressedImageRead(void *ptr, size_t sz, size_t count)
+{
+	size_t nread = 0, ntoread = sz * count;
+	unsigned long nreadSoFar = eizs.total_out;
+	int ret;
+
+	if (!eizs.next_in) {
+		eizs.avail_in = eicSize;
+		eizs.next_in = eiData;
+		ret = inflateInit2(&eizs,MAX_WBITS+16);
+		if (ret != Z_OK) {
+			fprintf(stderr,"inflateInit failed on reading compressed embedded image\n");
+			return 0;
+		}
+	}
+
+	if (eiReadPosition + ntoread > eiSize) {
+		fprintf(stderr,"Attempting to read beyond end of embedded image\n");
+		return 0;
+	}
+	// N.B. seeking backwards is as yet unimplemented (cuz all platforms,
+	// and hence all images, are little endian as of 2024). One way to
+	// implement this is to go back to the beginning and advance forward.
+	// This should be a fine strategy since seeking backwards is only done
+	// if the initial image magic number looks wrong, and the initial
+	// magic number is the first word in the image file.
+	assert(eiReadPosition >= nreadSoFar);
+	// to seek forward simply discard that much data
+	if (eiReadPosition > nreadSoFar) {
+		assert(eiReadPosition - nreadSoFar <= sz * count);
+		eizs.avail_out = eiReadPosition - nreadSoFar;
+		eizs.next_out = ptr;
+
+		// Decompress to fill ptr with eiReadPosition - nreadSoFar's worth
+		ret = inflate(&eizs, Z_NO_FLUSH);
+		switch (ret) {
+		case Z_NEED_DICT:
+		case Z_DATA_ERROR:
+		case Z_MEM_ERROR:
+			(void)inflateEnd(&eizs);
+			return nread;
+		}
+		nreadSoFar = eiReadPosition;
+	}
+	eizs.avail_out = ntoread;
+	eizs.next_out = ptr;
+
+	// Decompress to fill ptr with count's worth
+	ret = inflate(&eizs, Z_NO_FLUSH);
+	switch (ret) {
+	case Z_NEED_DICT:
+	case Z_DATA_ERROR:
+	case Z_MEM_ERROR:
+		(void)inflateEnd(&eizs);
+		return nread;
+	}
+	assert(eizs.total_out - nreadSoFar == sz * count);
+	eiReadPosition += ntoread;
+	return count;
+}
+
+
 #if !defined(min)
 # define min(a,b) ((a)<=(b)?(a):b)
 #endif
@@ -125,6 +204,8 @@ sqImageFileRead(void *ptr_arg, long sz, long count, sqImageFile f)
 
 	if (f == ImageIsEmbedded)
 		return sqEmbeddedImageRead(ptr,sz,count);
+	if (f == ImageIsEmbeddedAndCompressed)
+		return sqCompressedImageRead(ptr,sz,count);
 
 	/* read may refuse to write more than 2Gb-1.  At least on MacOS 10.13.6,
 	 * read craps out above 2Gb, so chunk the read into to 1Gb segments.
@@ -176,7 +257,8 @@ sqImageFileWrite(void *ptr_arg, size_t sz, size_t count, sqImageFile f)
 off_t
 sqImageFilePosition(sqImageFile f)
 {
-	if (f == ImageIsEmbedded)
+	if (f == ImageIsEmbedded
+	 || f == ImageIsEmbeddedAndCompressed)
 		return eiReadPosition;
 
 	off_t pos = lseek(f, 0, SEEK_CUR);
@@ -188,7 +270,8 @@ sqImageFilePosition(sqImageFile f)
 void
 sqImageFileSeek(sqImageFile f,off_t pos)
 {
-	if (f == ImageIsEmbedded)
+	if (f == ImageIsEmbedded
+	 || f == ImageIsEmbeddedAndCompressed)
 		eiReadPosition = pos;
 	else if (lseek(f, pos, SEEK_SET) < 0)
 		perror("sqImageFileSeek lseek");
@@ -197,7 +280,8 @@ sqImageFileSeek(sqImageFile f,off_t pos)
 void
 sqImageFileSeekEnd(sqImageFile f,off_t pos)
 {
-	if (f == ImageIsEmbedded)
+	if (f == ImageIsEmbedded
+	 || f == ImageIsEmbeddedAndCompressed)
 		eiReadPosition = eiSize;
 	else if (lseek(f, pos, SEEK_END) < 0)
 		perror("sqImageFileSeekEnd lseek");

platforms/unix/vm/sqUnixMain.c

@@ -2160,10 +2160,14 @@ imgInit(void)
 	void *embeddedImage;
 	if (handle && (embeddedImage = dlsym(handle,"embeddedImage"))) {
 		strcpy(shortImageName,dlsym(handle,"embeddedImageName"));
+		fd = ((char *)embeddedImage)[0] == GZIPMagic0 && ((char *)embeddedImage)[1] == GZIPMagic1
+				? ImageIsEmbeddedAndCompressed
+				: ImageIsEmbedded;
+		unsigned long *compressedSize = dlsym(handle,"embeddedCompressedDataSize");
 		noteEmbeddedImage(embeddedImage,
-						  *(unsigned long *)dlsym(handle,"embeddedImageSize"));
+						  *(unsigned long *)dlsym(handle,"embeddedImageSize"),
+						  compressedSize ? *compressedSize : 0);
 		dlclose(handle);
-		fd = ImageIsEmbedded;
 	}
 	else {