Skip to content

Commit

Permalink
Merge pull request #3761 from ferrys/3747-swift-and-compute-button
Browse files Browse the repository at this point in the history 
Updated storage identifier and swift container ID
  • Loading branch information
@ferrys
ferrys authored Apr 10, 2017
2 parents f74fe41 + dc30893 commit c0809e2
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 30 deletions.
4 changes: 1 addition & 3 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public DataAccess() {

// set by the user in glassfish-setup.sh if DEFFAULT_STORAGE_DRIVER_IDENTIFIER = swift
public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id");
public static final String DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS = System.getProperty("dataverse.files.swift-endpoint-start");
public static String swiftFileUri;
public static String swiftContainerUri;

Expand All @@ -58,8 +57,7 @@ public static DataFileIO getDataFileIO(DataFile df, DataAccessRequest req) throw
if (df.getStorageIdentifier().startsWith("file://")
|| (!df.getStorageIdentifier().matches("^[a-z][a-z]*://.*"))) {
return new FileAccessIO (df, req);
} else if (df.getStorageIdentifier().startsWith("swift://")
|| df.getStorageIdentifier().startsWith(DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {
} else if (df.getStorageIdentifier().startsWith("swift://")){
return new SwiftAccessIO(df, req);
} else if (df.getStorageIdentifier().startsWith("tmp://")) {
throw new IOException("DataAccess IO attempted on a temporary file that hasn't been permanently saved yet.");
Expand Down
83 changes: 56 additions & 27 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,10 @@ public void open(DataAccessOption... options) throws IOException {

// this is a Swift-specific override of the convenience method provided in the
// DataFileIO for copying a local Path (for ex., a temp file, into this DataAccess location):
@Override

@Override
public void copyPath(Path fileSystemPath) throws IOException {
long newFileSize = -1;
Properties p = getSwiftProperties();
String swiftEndPoint = p.getProperty("swift.default.endpoint");
String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint);

if (swiftFileObject == null || !this.canWrite()) {
open(DataAccessOption.WRITE_ACCESS);
Expand All @@ -131,11 +129,7 @@ public void copyPath(Path fileSystemPath) throws IOException {
try {
inputFile = fileSystemPath.toFile();

//@author Anuj Thakur
swiftFileObject.uploadObject(inputFile);
//After the files object is uploaded the identifier is changed.
logger.info(this.swiftFileName + " " + this.swiftFolderPath);
this.getDataFile().setStorageIdentifier(swiftDirectory + "/" + this.swiftFolderPath + "/" + this.swiftFileName);

newFileSize = inputFile.length();

Expand All @@ -153,6 +147,44 @@ public void copyPath(Path fileSystemPath) throws IOException {
setSize(newFileSize);

}
// @Override
// public void copyPath(Path fileSystemPath) throws IOException {
// long newFileSize = -1;
// Properties p = getSwiftProperties();
// String swiftEndPoint = p.getProperty("swift.default.endpoint");
// String swiftDirectory = p.getProperty("swift.swift_endpoint." + swiftEndPoint);

// if (swiftFileObject == null || !this.canWrite()) {
// open(DataAccessOption.WRITE_ACCESS);
// }

// File inputFile = null;

// try {
// inputFile = fileSystemPath.toFile();

// //@author Anuj Thakur
// swiftFileObject.uploadObject(inputFile);
// //After the files object is uploaded the identifier is changed.
// logger.info(this.swiftFileName + " " + this.swiftFolderPath);
// this.getDataFile().setStorageIdentifier( + this.swiftFolderPath + "/" + this.swiftFileName);

// newFileSize = inputFile.length();

// } catch (Exception ioex) {
// String failureMsg = ioex.getMessage();
// if (failureMsg == null) {
// failureMsg = "Swift AccessIO: Unknown exception occured while uploading a local file into a Swift StoredObject";
// }

// throw new IOException(failureMsg);
// }

// // if it has uploaded successfully, we can reset the size
// // of the object:
// setSize(newFileSize);

// }

@Override
public void delete() throws IOException {
Expand Down Expand Up @@ -212,26 +244,20 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
String swiftEndPoint = null;
String swiftContainer = null;

//
// if (storageIdentifier.startsWith("swift://")) {
if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {

if (storageIdentifier.startsWith("swift://")) {
// This is a call on an already existing swift object.

//String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);
//The Storage identifier now is the Object store service endpoint
//followed by the endpoint named container
String[] swiftStorageTokens = storageIdentifier.substring(46).split("/", 2);

if (swiftStorageTokens.length != 2) {

String[] swiftStorageTokens = storageIdentifier.substring(8).split(":", 3);

if (swiftStorageTokens.length != 3) {
// bad storage identifier
throw new IOException("SwiftAccessIO: invalid swift storage token: " + storageIdentifier);
}

Properties p = getSwiftProperties();
swiftEndPoint = p.getProperty("swift.default.endpoint");

swiftContainer = swiftStorageTokens[0];
swiftFileName = swiftStorageTokens[1];
swiftEndPoint = swiftStorageTokens[0];
swiftContainer = swiftStorageTokens[1];
swiftFileName = swiftStorageTokens[2];

if (swiftEndPoint == null || swiftContainer == null || swiftFileName == null
|| "".equals(swiftEndPoint) || "".equals(swiftContainer) || "".equals(swiftFileName)) {
Expand All @@ -246,12 +272,15 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
} else if (this.isWriteAccess) {
Properties p = getSwiftProperties();
swiftEndPoint = p.getProperty("swift.default.endpoint");
swiftFolderPath = this.getDataFile().getOwner().getDisplayName();
//swiftFolderPath = this.getDataFile().getOwner().getIdentifier(); /* TODO: ? */

//swiftFolderPath = this.getDataFile().getOwner().getDisplayName();
String swiftFolderPathSeparator = "_";
swiftFolderPath = this.getDataFile().getOwner().getAuthority().replace(this.getDataFile().getOwner().getDoiSeparator(), swiftFolderPathSeparator) +
swiftFolderPathSeparator + this.getDataFile().getOwner().getIdentifier();
swiftFileName = storageIdentifier;
//swiftFileName = this.getDataFile().getDisplayName();
//Storage Identifier is now updated after the object is uploaded on Swift.
// this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftContainer+":"+swiftFileName);
this.getDataFile().setStorageIdentifier("swift://"+swiftEndPoint+":"+swiftFolderPath+":"+swiftFileName);
} else {
throw new IOException("SwiftAccessIO: unknown access mode.");
}
Expand All @@ -272,7 +301,7 @@ private StoredObject initializeSwiftFileObject(boolean writeAccess) throws IOExc
*/
Container dataContainer;

if (storageIdentifier.startsWith(DataAccess.DEFAULT_SWIFT_ENDPOINT_START_CHARACTERS)) {
if (storageIdentifier.startsWith("swift://")) {
dataContainer = account.getContainer(swiftContainer);
} else {
dataContainer = account.getContainer(swiftFolderPath); //changed from swiftendpoint
Expand Down

0 comments on commit c0809e2

Please sign in to comment.