feat(server): faster geodata import (#14241)

* faster geodata import * revert logging change * unlogged tables * leave spare connection * use expression index instead of generated column * do btree indexing with others
immich-app · Nov 20, 2024 · ad510dd · ad510dd
1 parent a3712e4
commit ad510dd
Show file tree

Hide file tree

Showing 4 changed files with 214 additions and 138 deletions.
diff --git a/server/src/entities/geodata-places.entity.ts b/server/src/entities/geodata-places.entity.ts
@@ -14,12 +14,41 @@ export class GeodataPlacesEntity {
   @Column({ type: 'float' })
   latitude!: number;
 
-  // @Column({
-  //   generatedType: 'STORED',
-  //   asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)',
-  //   type: 'earth',
-  // })
-  // earthCoord!: unknown;
+  @Column({ type: 'char', length: 2 })
+  countryCode!: string;
+
+  @Column({ type: 'varchar', length: 20, nullable: true })
+  admin1Code!: string;
+
+  @Column({ type: 'varchar', length: 80, nullable: true })
+  admin2Code!: string;
+
+  @Column({ type: 'varchar', nullable: true })
+  admin1Name!: string;
+
+  @Column({ type: 'varchar', nullable: true })
+  admin2Name!: string;
+
+  @Column({ type: 'varchar', nullable: true })
+  alternateNames!: string;
+
+  @Column({ type: 'date' })
+  modificationDate!: Date;
+}
+
+@Entity('geodata_places_tmp', { synchronize: false })
+export class GeodataPlacesTempEntity {
+  @PrimaryColumn({ type: 'integer' })
+  id!: number;
+
+  @Column({ type: 'varchar', length: 200 })
+  name!: string;
+
+  @Column({ type: 'float' })
+  longitude!: number;
+
+  @Column({ type: 'float' })
+  latitude!: number;
 
   @Column({ type: 'char', length: 2 })
   countryCode!: string;

diff --git a/server/src/entities/natural-earth-countries.entity.ts b/server/src/entities/natural-earth-countries.entity.ts
@@ -2,7 +2,25 @@ import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm';
 
 @Entity('naturalearth_countries', { synchronize: false })
 export class NaturalEarthCountriesEntity {
-  @PrimaryGeneratedColumn()
+  @PrimaryGeneratedColumn('identity', { generatedIdentity: 'ALWAYS' })
+  id!: number;
+
+  @Column({ type: 'varchar', length: 50 })
+  admin!: string;
+
+  @Column({ type: 'varchar', length: 3 })
+  admin_a3!: string;
+
+  @Column({ type: 'varchar', length: 50 })
+  type!: string;
+
+  @Column({ type: 'polygon' })
+  coordinates!: string;
+}
+
+@Entity('naturalearth_countries_tmp', { synchronize: false })
+export class NaturalEarthCountriesTempEntity {
+  @PrimaryGeneratedColumn('identity', { generatedIdentity: 'ALWAYS' })
   id!: number;
 
   @Column({ type: 'varchar', length: 50 })

diff --git a/server/src/migrations/1732072134943-NaturalEarthCountriesIdentityColumn.ts b/server/src/migrations/1732072134943-NaturalEarthCountriesIdentityColumn.ts
@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class NaturalEarthCountriesIdentityColumn1732072134943 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE naturalearth_countries ALTER id DROP DEFAULT`);
+    await queryRunner.query(`DROP SEQUENCE naturalearth_countries_id_seq`);
+    await queryRunner.query(`ALTER TABLE naturalearth_countries ALTER id ADD GENERATED ALWAYS AS IDENTITY`);
+
+    // same as ll_to_earth, but with explicit schema to avoid weirdness and allow it to work in expression indices
+    await queryRunner.query(`
+      CREATE FUNCTION ll_to_earth_public(latitude double precision, longitude double precision) RETURNS public.earth PARALLEL SAFE IMMUTABLE STRICT LANGUAGE SQL AS $$
+        SELECT public.cube(public.cube(public.cube(public.earth()*cos(radians(latitude))*cos(radians(longitude))),public.earth()*cos(radians(latitude))*sin(radians(longitude))),public.earth()*sin(radians(latitude)))::public.earth
+    $$`);
+
+    await queryRunner.query(`ALTER TABLE geodata_places DROP COLUMN "earthCoord"`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE naturalearth_countries ALTER id DROP GENERATED`);
+    await queryRunner.query(`CREATE SEQUENCE naturalearth_countries_id_seq`);
+    await queryRunner.query(
+      `ALTER TABLE naturalearth_countries ALTER id SET DEFAULT nextval('naturalearth_countries_id_seq'::regclass)`,
+    );
+    await queryRunner.query(`DROP FUNCTION ll_to_earth_public`);
+    await queryRunner.query(
+      `ALTER TABLE "geodata_places" ADD "earthCoord" earth GENERATED ALWAYS AS (ll_to_earth(latitude, longitude)) STORED`,
+    );
+  }
+}