Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated storage identifier and swift container ID #3761

Merged
merged 1 commit into from
Apr 10, 2017
Merged

Updated storage identifier and swift container ID #3761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public DataAccess() {

// set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift
public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id");
public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start");
public static String swiftFileUri;
public static String swiftContainerUri;

Expand All @@ -58,8 +57,7 @@ public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throw
if (df.getStorageIdentifier().startsWith("file://")
|| (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) {
return new FileAccessIO (df, req);
} else if (df.getStorageIdentifier().startsWith("swift://")
|| df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {
} else if (df.getStorageIdentifier().startsWith("swift://")){
return new SwiftAccessIO(df, req);
} else if (df.getStorageIdentifier().startsWith("tmp://")) {
throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet.");
Expand Down
83 changes: 56 additions & 27 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,10 @@ public void open(DataAccessOption... options) throws IOException {

// this is a Swift-specific override of the convenience method provided in the
// DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location):
@Override

@Override
public void copyPath(Path fileSystemPath) throws IOException {
long newFileSize = -1;
Properties p = getSwiftProperties();
String swiftEndPoint = p.getProperty("swift.default.endpoint");
String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint);

if (swiftFileObject == null || !this.canWrite()) {
open(DataAccessOption.WRITE_ACCESS);
Expand All @@ -131,11 +129,7 @@ public void copyPath(Path fileSystemPath) throws IOException {
try {
inputFile = fileSystemPath.toFile();

//@author Anuj Thakur
swiftFileObject.uploadObject(inputFile);
//After the files object is uploaded the identifier is changed.
logger.info(this.swiftFileName + " " + this.swiftFolderPath);
this.getDataFile().setStorageIdentifier(swiftDirectory + "/" + this.swiftFolderPath + "/" + this.swiftFileName);

newFileSize = inputFile.length();

Expand All @@ -153,6 +147,44 @@ public void copyPath(Path fileSystemPath) throws IOException {
setSize(newFileSize);

}
// @Override
// public void copyPath(Path fileSystemPath) throws IOException {
// long newFileSize = -1;
// Properties p = getSwiftProperties();
// String swiftEndPoint = p.getProperty("swift.default.endpoint");
// String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint);

// if (swiftFileObject == null || !this.canWrite()) {
// open(DataAccessOption.WRITE_ACCESS);
// }

// File inputFile = null;

// try {
// inputFile = fileSystemPath.toFile();

// //@author Anuj Thakur
// swiftFileObject.uploadObject(inputFile);
// //After the files object is uploaded the identifier is changed.
// logger.info(this.swiftFileName + " " + this.swiftFolderPath);
// this.getDataFile().setStorageIdentifier( + this.swiftFolderPath + "/" + this.swiftFileName);

// newFileSize = inputFile.length();

// } catch (Exception ioex) {
// String failureMsg = ioex.getMessage();
// if (failureMsg == null) {
// failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject";
// }

// throw new IOException(failureMsg);
// }

// // if it has uploaded successfully, we can reset the size
// // of the object:
// setSize(newFileSize);

// }

@Override
public void delete() throws IOException {
Expand Down Expand Up @@ -212,26 +244,20 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
String swiftEndPoint = null;
String swiftContainer = null;

//
// if (storageIdentifier.startsWith("swift://")) {
if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {

if (storageIdentifier.startsWith("swift://")) {
// This is a call on an already existing swift object.

//String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);
//The Storage identifier now is the Object store service endpoint
//followed by the endpoint named container
String[] swiftStorageTokens = storageIdentifier.substring(46).split("/", 2);

if (swiftStorageTokens.length != 2) {

String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);

if (swiftStorageTokens.length != 3) {
// bad storage identifier
throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
}

Properties p = getSwiftProperties();
swiftEndPoint = p.getProperty("swift.default.endpoint");

swiftContainer = swiftStorageTokens[0];
swiftFileName = swiftStorageTokens[1];
swiftEndPoint = swiftStorageTokens[0];
swiftContainer = swiftStorageTokens[1];
swiftFileName = swiftStorageTokens[2];

if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null
|| "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) {
Expand All @@ -246,12 +272,15 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
} else if (this.isWriteAccess) {
Properties p = getSwiftProperties();
swiftEndPoint = p.getProperty("swift.default.endpoint");
swiftFolderPath = this.getDataFile().getOwner().getDisplayName();
//swiftFolderPath = this.getDataFile().getOwner().getIdentifier(); /* TODO: ? */

//swiftFolderPath = this.getDataFile().getOwner().getDisplayName();
String swiftFolderPathSeparator = "_";
swiftFolderPath = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator) +
swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier();
swiftFileName = storageIdentifier;
//swiftFileName = this.getDataFile().getDisplayName();
//Storage Identifier is now updated after the object is uploaded on Swift.
// this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftContainer+":"+swiftFileName);
this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftFolderPath+":"+swiftFileName);
} else {
throw new IOException("SwiftAccessIO: unknown access mode.");
}
Expand All @@ -272,7 +301,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
*/
Container dataContainer;

if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {
if (storageIdentifier.startsWith("swift://")) {
dataContainer = account.getContainer(swiftContainer);
} else {
dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint
Expand Down