Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(dataSource): rename FileStore to FileSystem #451

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/actions/confWriter/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ inputs:
runs:
using: "composite" # Mandatory parameter
steps:
- if: inputs.DB-name=='FileStore'
name: save config for FileStore
- if: inputs.DB-name=='FileSystem'
name: save config for FileSystem
shell: bash
run: |
cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties"

- if: inputs.DB-name=='FileStore'
name: save config for FileStore
- if: inputs.DB-name=='FileSystem'
name: save config for FileSystem
shell: bash
run: |
cp -f "${{ inputs.Root-Dir-Path }}/conf/config.properties" "${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties"
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/dbConfWriter/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@ runs:
paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
statements: s/#storageEngineList=127.0.0.1#8086/storageEngineList=127.0.0.1#8086/g

- if: inputs.DB-name == 'FileStore'
- if: inputs.DB-name == 'FileSystem'
name: Modify IGinX Config
uses: ./.github/actions/edit
with:
paths: ${{ inputs.Root-Dir-Path }}/conf/config.properties
statements: |
s/^#storageEngineList=127.0.0.1#6667#filestore/storageEngineList=127.0.0.1#6667#filestore/g
s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g
s#dir=data#dir=${PWD}/test/iginx_mn#g
s/data.config.write.buffer.size=104857600/data.config.write.buffer.size=1048576/g
s/write.buffer.size=104857600/write.buffer.size=1048576/g
Expand Down
6 changes: 3 additions & 3 deletions .github/actions/dbRunner/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -169,19 +169,19 @@ runs:
paths: conf/config.properties
statements: s|^#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path|storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#has_data=false#meta_properties_path=${{ steps.mysql-properties.outputs.path }}|g

- if: inputs.DB-name=='FileStore'
- if: inputs.DB-name=='FileSystem'
name: Run DB
shell: bash
run: |
cp -f "${GITHUB_WORKSPACE}/conf/config.properties" "${GITHUB_WORKSPACE}/conf/config.properties.bak"

- if: inputs.DB-name == 'FileStore'
- if: inputs.DB-name == 'FileSystem'
name: Modify IGinX Config
uses: ./.github/actions/edit
with:
paths: conf/config.properties
statements: |
s/^#storageEngineList=127.0.0.1#6667#filestore/storageEngineList=127.0.0.1#6667#filestore/g
s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#6667#filesystem/g
s#dir=data#dir=${{ steps.project.outputs.workspace }}/test/iginx_mn#g
s#dummy_dir=dummy#dummy_dir=${{ steps.project.outputs.workspace }}/test/mn#g
s/data.config.write.buffer.size=104857600/data.config.write.buffer.size=1048576/g
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ cp -f conf/config.properties.bak $7

sed -i"" -e "s/^storageEngineList=127.0.0.1#6667#iotdb12/#storageEngineList=127.0.0.1#6667#iotdb12/" $7

sed -i"" -e "s/^#storageEngineList=127.0.0.1#6667#filestore/storageEngineList=127.0.0.1#$1#filestore/g" $7
sed -i"" -e "s/^#storageEngineList=127.0.0.1#6667#filesystem/storageEngineList=127.0.0.1#$1#filesystem/g" $7

sed -i"" -e "s/#iginx_port=6888#/#iginx_port=$2#/g" $7

Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/test/cli/test_outfile.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ for file in test/src/test/resources/fileReadAndWrite/byteDummy/*; do
mv "$file" "${file}.ext"
done

bash -c "echo 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filestore"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;' | ${SCRIPT_COMMAND}"
bash -c "echo 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filesystem"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;' | ${SCRIPT_COMMAND}"

bash -c "echo 'select * from byteDummy into outfile "'"test/src/test/resources/fileReadAndWrite/byteStreamExport"'" as stream;' | ${SCRIPT_COMMAND}"

db_name=$1

if [[ "$db_name" != "FileStore" ]]; then
if [[ "$db_name" != "FileSystem" ]]; then
exit 0
fi

Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/test/cli/test_outfile_macos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@ for file in test/src/test/resources/fileReadAndWrite/byteDummy/*; do
mv "$file" "${file}.ext"
done

bash -c "echo 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filestore"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;' | ${SCRIPT_COMMAND}"
bash -c "echo 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filesystem"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;' | ${SCRIPT_COMMAND}"

bash -c "echo 'select * from byteDummy into outfile "'"test/src/test/resources/fileReadAndWrite/byteStreamExport"'" as stream;' | ${SCRIPT_COMMAND}"

db_name=$1

if [[ "$db_name" != "FileStore" ]]; then
if [[ "$db_name" != "FileSystem" ]]; then
exit 0
fi

Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/test/cli/test_outfile_windows.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@ for file in test/src/test/resources/fileReadAndWrite/byteDummy/*; do
mv "$file" "${file}.ext"
done

bash -c "client/target/iginx-client-$2/sbin/start_cli.bat -e 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filestore"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;'"
bash -c "client/target/iginx-client-$2/sbin/start_cli.bat -e 'ADD STORAGEENGINE ("'"127.0.0.1"'", 6670, "'"filesystem"'", "'"dummy_dir:test/src/test/resources/fileReadAndWrite/byteDummy,iginx_port:6888,has_data:true,is_read_only:true"'");show columns byteDummy.*;'"

bash -c "client/target/iginx-client-$2/sbin/start_cli.bat -e 'select * from byteDummy into outfile "'"test/src/test/resources/fileReadAndWrite/byteStreamExport"'" as stream;'"

db_name=$1

if [[ "$db_name" != "FileStore" ]]; then
if [[ "$db_name" != "FileSystem" ]]; then
exit 0
fi

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/DB-CE.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
description: "The database to run the test on"
type: string
required: false
default: '["IoTDB12", "InfluxDB", "FileStore", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'

env:
FUNCTEST: NewSessionIT,SQLCompareIT,TagIT,RestIT,TransformIT,UDFIT,RestAnnotationIT,SQLSessionIT,SQLSessionPoolIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/remote-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ on:
description: "The database to run the test on"
type: string
required: false
default: '["IoTDB12", "FileStore"]'
default: '["IoTDB12", "FileSystem"]'

env:
FUNCTEST: RemoteUDFIT
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/standalone-test-pushdown.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
description: "The database to run the test on"
type: string
required: false
default: '["IoTDB12", "InfluxDB", "FileStore", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'

jobs:
Union-DB-Test-Push_Down:
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/standalone-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
description: "The database to run the test on"
type: string
required: false
default: '["IoTDB12", "InfluxDB", "FileStore", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'

jobs:
Union-DB-Test:
Expand Down Expand Up @@ -106,12 +106,12 @@ jobs:
shell: client-before

# TODO: extract it to a separate job to test
# large image export only tested in FileStore
# large image export only tested in FileSystem
- name: Test Client Export File
if: always()
shell: bash
run: |
if [[ "${{ matrix.DB-name }}" == "FileStore" ]]; then
if [[ "${{ matrix.DB-name }}" == "FileSystem" ]]; then
mvn test -q -Dtest=ExportFileIT -DfailIfNoTests=false -P-format
else
mvn test -q -Dtest=ExportFileIT#checkExportByteStream -DfailIfNoTests=false -P-format
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tpc-h.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ on:
description: "The database to run the test on"
type: string
required: false
default: '["IoTDB12", "InfluxDB", "FileStore", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'

jobs:
TPC-H-Test:
Expand Down
6 changes: 3 additions & 3 deletions assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
</dependency>
<dependency>
<groupId>cn.edu.tsinghua</groupId>
<artifactId>filestore</artifactId>
<artifactId>filesystem</artifactId>
</dependency>
<dependency>
<groupId>cn.edu.tsinghua</groupId>
Expand Down Expand Up @@ -175,8 +175,8 @@
<value>#storageEngineList=127.0.0.1#6667#iotdb</value>
</replacement>
<replacement>
<token>#storageEngineList=127.0.0.1#6667#filestore.*</token>
<value>storageEngineList=127.0.0.1#6667#filestore#dir=data#iginx_port=6888</value>
<token>#storageEngineList=127.0.0.1#6667#filesystem.*</token>
<value>storageEngineList=127.0.0.1#6667#filesystem#dir=data#iginx_port=6888</value>
</replacement>
<replacement>
<token>UD[ATS]F,.*,.*,.*[.]py</token>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<include>cn.edu.tsinghua:filestore</include>
<include>cn.edu.tsinghua:filesystem</include>
</includes>
<sources>
<includeModuleDirectory>false</includeModuleDirectory>
<fileSets>
<fileSet>
<directory>${project.build.directory}/filestore-${project.version}</directory>
<directory>${project.build.directory}/filesystem-${project.version}</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
</fileSets>
Expand Down
2 changes: 1 addition & 1 deletion assembly/src/assembly/include.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<componentDescriptors>
<componentDescriptor>src/assembly/component/iginx-core.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-iotdb12-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-filestore-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-filesystem-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/include-zookeeper.xml</componentDescriptor>
</componentDescriptors>
<fileSets>
Expand Down
2 changes: 1 addition & 1 deletion assembly/src/assembly/server.xml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
<includeBaseDirectory>false</includeBaseDirectory>
<componentDescriptors>
<componentDescriptor>src/assembly/component/iginx-core.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-filestore-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-filesystem-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-influxdb-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-iotdb12-driver.xml</componentDescriptor>
<componentDescriptor>src/assembly/component/iginx-mongodb-driver.xml</componentDescriptor>
Expand Down
4 changes: 2 additions & 2 deletions conf/config.properties
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ password=root
# 数据库列表,使用','分隔不同实例
storageEngineList=127.0.0.1#6667#iotdb12#username=root#password=root#sessionPoolSize=20#has_data=false#is_read_only=false
#storageEngineList=127.0.0.1#8086#influxdb#url=http://localhost:8086/#token=your-token#organization=your-organization#has_data=false
#storageEngineList=127.0.0.1#6667#filestore#iginx_port=6888#has_data=false#is_read_only=false#dir=data#data.config.write.buffer.size=104857600#data.config.write.buffer.timeout=0#dummy_dir=dummy#dummy.struct=LegacyFilesystem#dummy.config.chunk_size_in_bytes=1048576#dummy.config.memory_pool_size=100#client.connectPool.maxTotal=100
#storageEngineList=127.0.0.1#6667#filesystem#iginx_port=6888#has_data=false#is_read_only=false#dir=data#data.config.write.buffer.size=104857600#data.config.write.buffer.timeout=0#dummy_dir=dummy#dummy.struct=LegacyFilesystem#dummy.config.chunk_size_in_bytes=1048576#dummy.config.memory_pool_size=100#client.connectPool.maxTotal=100
#storageEngineList=127.0.0.1#5432#relational#engine=postgresql#username=postgres#password=postgres#has_data=false
#storageEngineList=127.0.0.1#3306#relational#engine=mysql#username=root#password=mysql#has_data=false#meta_properties_path=your-meta-properties-path
#storageEngineList=127.0.0.1#27017#mongodb#uri="mongodb://127.0.0.1:27017/?maxPoolSize=200&maxIdleTimeMS=60000&waitQueueTimeoutMS=50000"#has_data=false#schema.sample.size=1000#dummy.sample.size=0
Expand All @@ -51,7 +51,7 @@ defaultUDFDir=udf_funcs
replicaNum=0

# 底层数据库类名
databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,filestore=cn.edu.tsinghua.iginx.filestore.FileStoreStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage
databaseClassNames=iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage

# 内存任务执行线程池
memoryTaskThreadPoolSize=200
Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public class IginxWorker implements IService.Iface {
private static final Config config = ConfigDescriptor.getInstance().getConfig();

private IginxWorker() {
// if there are new local filestore in conf, add them to cluster.
// if there are new local filesystem in conf, add them to cluster.
if (!addLocalStorageEngineMetas()) {
LOGGER.error("there are no valid storage engines!");
System.exit(-1);
Expand Down Expand Up @@ -459,8 +459,8 @@ private void addStorageEngineMetas(
}
}

// init local filestore before adding to meta
// exclude remote filestore
// init local filesystem before adding to meta
// exclude remote filesystem
List<StorageEngineMeta> localMetas = new ArrayList<>();
List<StorageEngineMeta> otherMetas = new ArrayList<>();
for (StorageEngineMeta meta : storageEngineMetas) {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/cn/edu/tsinghua/iginx/conf/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public class Config {
private TimePrecision timePrecision = TimePrecision.NS;

private String databaseClassNames =
"iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filestore=cn.edu.tsinghua.iginx.filestore.FileStorage";
"iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage";

private String policyClassName = "cn.edu.tsinghua.iginx.policy.naive.NaivePolicy";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ private void loadPropsFromFile() {
config.setDatabaseClassNames(
properties.getProperty(
"databaseClassNames",
"iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filestore=cn.edu.tsinghua.iginx.filestore.FileStorage"));
"iotdb12=cn.edu.tsinghua.iginx.iotdb.IoTDBStorage,influxdb=cn.edu.tsinghua.iginx.influxdb.InfluxDBStorage,relational=cn.edu.tsinghua.iginx.relational.RelationalStorage,mongodb=cn.edu.tsinghua.iginx.mongodb.MongoDBStorage,redis=cn.edu.tsinghua.iginx.redis.RedisStorage,filesystem=cn.edu.tsinghua.iginx.filesystem.FileSystemStorage"));

config.setPolicyClassName(
properties.getProperty(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public class Constants {

public static final String DATA_PREFIX = "data_prefix";

// especially for embedded storage engines: filestore
// especially for embedded storage engines: filesystem
public static final String EMBEDDED_PREFIX = "embedded_prefix";

public static final String SCHEMA_PREFIX = "schema_prefix";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
public class StorageEngineUtils {

public static boolean isEmbeddedStorageEngine(StorageEngineType type) {
return type.equals(StorageEngineType.filestore);
return type.equals(StorageEngineType.filesystem);
}

private static boolean isDirValid(String dir) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ private void runParamTest(
extraParam.put(DUMMY_DIR, dummyDir);
}
if (valid) {
assertTrue(checkEmbeddedStorageExtraParams(StorageEngineType.filestore, extraParam));
assertTrue(checkEmbeddedStorageExtraParams(StorageEngineType.filesystem, extraParam));
} else {
assertFalse(checkEmbeddedStorageExtraParams(StorageEngineType.filestore, extraParam));
assertFalse(checkEmbeddedStorageExtraParams(StorageEngineType.filesystem, extraParam));
}
}

Expand Down
Loading
Loading