Skip to content

Commit

Permalink
Add support for environment variables in minimal configuration archives
Browse files Browse the repository at this point in the history 
If archiving a minimal config and environment variables are used, it must be possible for all the needed variables to be resolved and any files pointed to should also exist at the time of archiving.

The values of the environment will be written into the saved configuration in the environment section of the configuration file. Ths configuration stored thus may be different to the initial config being passed for archiving.

Needs tests adding and futher discussion in the TSC meetings.

Signed-off-by: Kevin Wheatley <kevin.wheatley@framestore.com>
  • Loading branch information
@KevinJW
KevinJW committed Jan 22, 2024
1 parent 699d610 commit 7f1a112
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 54 deletions.
2 changes: 1 addition & 1 deletion include/OpenColorIO/OpenColorIO.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1504,7 +1504,7 @@ class OCIOEXPORT Config
*
* \return bool Archivable if true.
*/
bool isArchivable() const;
bool isArchivable(bool minimal) const;

/**
* \brief Archive the config and its LUTs into the specified output stream.
Expand Down
9 changes: 5 additions & 4 deletions src/OpenColorIO/Config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5472,7 +5472,7 @@ ConfigIOProxyRcPtr Config::getConfigIOProxy() const
return getImpl()->m_context->getConfigIOProxy();
}

bool Config::isArchivable() const
bool Config::isArchivable(bool minimal) const
{
ConstContextRcPtr context = getCurrentContext();

Expand All @@ -5485,7 +5485,7 @@ bool Config::isArchivable() const
}

// Utility lambda to check the following criteria.
auto validatePathForArchiving = [](const std::string & path)
auto validatePathForArchiving = [&minimal](const std::string & path)
{
// Using the normalized path.
const std::string normPath = pystring::os::path::normpath(path);
Expand All @@ -5496,8 +5496,9 @@ bool Config::isArchivable() const
pystring::startswith(normPath, "..") ||
// 3) A context variable may not be located at the start of the path.
(ContainsContextVariables(path) && //TODO: if we can resolve context, do so and validate path to file
(StringUtils::Find(path, "$") == 0 ||
StringUtils::Find(path, "%") == 0)))
(!minimal &&
(StringUtils::Find(path, "$") == 0 ||
StringUtils::Find(path, "%") == 0))))
{
return false;
}
Expand Down
98 changes: 70 additions & 28 deletions src/OpenColorIO/OCIOZArchive.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "Platform.h"
#include "utils/StringUtils.h"
#include "transforms/FileTransform.h"
#include "Logging.h"

#include "OCIOZArchive.h"

Expand Down Expand Up @@ -183,6 +184,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
if (!possibleFormats.empty())
{
// The extension is supported. Add the current file to the OCIOZ archive.
std::ostringstream logMessage;
logMessage << "Adding file: " << absPath;
LogInfo(logMessage.str());
if (mz_zip_writer_add_path(
archiver, absPath.c_str(),
configWorkingDirectory, 0, 1) != MZ_OK)
Expand All @@ -201,9 +205,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
}
}

void addReferencedFiles(void * archiver, const Config & config)
ContextRcPtr addReferencedFiles(void * archiver, const ConfigRcPtr & config)
{
ConstContextRcPtr context = config.getCurrentContext();
ConstContextRcPtr context = config->getCurrentContext();
ContextRcPtr ctxFilepath = Context::Create();
ctxFilepath->setSearchPath(context->getSearchPath());
ctxFilepath->setWorkingDir(context->getWorkingDir());
Expand All @@ -212,20 +216,42 @@ void addReferencedFiles(void * archiver, const Config & config)
auto prefixLength = std::string(context->getWorkingDir()).length() + 1; // +1 add trailing '/' TODO: improve this

std::set<std::string> files;
config.GetAllFileReferences(files);
config->GetAllFileReferences(files);
std::set<std::string> added_files;
for (const auto &file : files)
{
const std::string resolvedPath = context->resolveFileLocation(file.c_str(), ctxFilepath);
const std::string relativePath = resolvedPath.substr(prefixLength);
std::ostringstream logMessage;

auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
if (returnCode != MZ_OK)
if (added_files.find(relativePath) == added_files.end())
{
std::ostringstream os;
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
throw Exception(os.str().c_str());
logMessage << "Adding file: " << file << " -> " << relativePath;
LogInfo(logMessage.str());
auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
if (returnCode != MZ_OK)
{
std::ostringstream os;
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
throw Exception(os.str().c_str());
}
}
else
{
logMessage << "Skipping already added file: " << file << " -> " << relativePath;
LogInfo(logMessage.str());
}
added_files.insert(relativePath);
}

const auto variableCount = ctxFilepath->getNumStringVars();
for (auto i = 0; i != variableCount; ++i)
{
std::ostringstream logMessage;
logMessage << "Used Variable: " << ctxFilepath->getStringVarNameByIndex(i) << " -> " << ctxFilepath->getStringVarByIndex(i);
LogInfo(logMessage.str());
}
return ctxFilepath;
}

//////////////////////////////////////////////////////////////////////////////////////
Expand All @@ -250,8 +276,9 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
mz_zip_file file_info;

flags = EnvironmentOverride(flags);
const bool minimal = HasFlag(flags, ARCHIVE_FLAGS_MINIMAL);

if (!config.isArchivable()) // TODO: pass in flags?
if (!config.isArchivable(minimal)) // TODO: pass in flags?
{
std::ostringstream os;
os << "Config is not archivable.";
Expand All @@ -261,11 +288,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
// Initialize.
memset(&file_info, 0, sizeof(file_info));

// Retrieve and store the config as string.
std::stringstream ss;
config.serialize(ss);
std::string configStr = ss.str();

// Write zip to memory stream.
#if MZ_VERSION_BUILD >= 040000
write_mem_stream = mz_stream_mem_create();
Expand Down Expand Up @@ -303,10 +325,38 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
// Open the in-memory zip.
if (mz_zip_writer_open(archiver, write_mem_stream, 0) == MZ_OK)
{
ConfigRcPtr editiableConfig = config.createEditableCopy(); // TODO: is this a little heavy handed just so we can modify the envronment?
///////////////////////
// Adding LUT files //
///////////////////////
if (minimal)
{
ContextRcPtr archivedContext = addReferencedFiles(archiver, editiableConfig);

// Need to make sure the used evironment context is in the config
const auto variableCount = archivedContext->getNumStringVars();
for (auto i = 0; i != variableCount; ++i)
{
editiableConfig->addEnvironmentVar(archivedContext->getStringVarNameByIndex(i), archivedContext->getStringVarByIndex(i));
}
}
else
{
// Add all supported files to in-memory zip from any directories under working directory.
// (recursive)
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);
}

//////////////////////////////
// Adding config to archive //
//////////////////////////////
// Use a hardcoded name for the config's filename inside the archive.
std::string configFullname = std::string(OCIO_CONFIG_DEFAULT_NAME) +
std::string(OCIO_CONFIG_DEFAULT_FILE_EXT);

std::stringstream ss;
editiableConfig->serialize(ss);
std::string configStr = ss.str();
// Get config string size.
int32_t configSize = (int32_t) configStr.size();

Expand All @@ -316,11 +366,13 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
file_info.version_madeby = MZ_VERSION_MADEBY;
file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE;
file_info.flag = MZ_ZIP_FLAG_UTF8;
file_info.uncompressed_size = configSize;
file_info.uncompressed_size = configSize; // Retrieve and store the config as string.

constexpr uint32_t posix_attrib = 0100644; // File with -rw-r--r-- permissions
constexpr uint32_t msdos_attrib = 0200; // MSDOS equivalent
file_info.external_fa = msdos_attrib;
file_info.external_fa |= (posix_attrib << 16);

//////////////////////////////
// Adding config to archive //
//////////////////////////////
int32_t written = 0;
// Opens an entry in the in-memory zip file for writing.
if (mz_zip_writer_entry_open(archiver, &file_info) == MZ_OK)
Expand All @@ -343,16 +395,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
throw Exception(os.str().c_str());
}

///////////////////////
// Adding LUT files //
///////////////////////
// Add all supported files to in-memory zip from any directories under working directory.
// (recursive)
if (HasFlag(flags, ARCHIVE_FLAGS_MINIMAL))
addReferencedFiles(archiver, config);
else
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);

std::ostringstream comment;
comment << "Configuration written by archiveConfig() OCIO: " << GetVersion();
mz_zip_writer_set_comment(archiver, comment.str().c_str());
Expand Down
3 changes: 2 additions & 1 deletion src/apps/ociocheck/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,6 +504,7 @@ int main(int argc, const char **argv)

std::string cacheID;
bool isArchivable = false;
const bool minimal = false;
try
{
LogGuard logGuard;
Expand All @@ -512,7 +513,7 @@ int main(int argc, const char **argv)
std::cout << logGuard.output();

cacheID = config->getCacheID();
isArchivable = config->isArchivable();
isArchivable = config->isArchivable(minimal);

// Passed if there are no Error level logs.
StringUtils::StringVec svec = StringUtils::SplitByLines(logGuard.output());
Expand Down
41 changes: 21 additions & 20 deletions tests/cpu/OCIOZArchive_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)

std::istringstream iss;
iss.str(CONFIG);
const bool minimal = false;

OCIO::ConfigRcPtr cfg;
OCIO_CHECK_NO_THROW(cfg = OCIO::Config::CreateFromStream(iss)->createEditableCopy());
Expand All @@ -95,74 +96,74 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)

// Valid search path.
cfg->setSearchPath("luts");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts/myluts1)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts\myluts1)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

// Valid Search path starting with "./" or ".\".
cfg->setSearchPath(R"(./myLuts)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(.\myLuts)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

// Valid search path starting with "./" or ".\" and a context variable.
cfg->setSearchPath(R"(./$SHOT/myluts)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(.\$SHOT\myluts)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts/$SHOT)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts/$SHOT/luts1)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts\$SHOT)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts\$SHOT\luts1)");
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));

/*
* Illegal scenarios
*/

// Illegal search path starting with "..".
cfg->setSearchPath(R"(luts:../luts)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts:..\myLuts)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

// Illegal search path starting with a context variable.
cfg->setSearchPath(R"(luts:$SHOT)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

// Illegal search path with absolute path.
cfg->setSearchPath(R"(luts:/luts)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

cfg->setSearchPath(R"(luts:/$SHOT)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

#ifdef _WIN32
cfg->clearSearchPaths();
cfg->addSearchPath(R"(C:\luts)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

cfg->clearSearchPaths();
cfg->addSearchPath(R"(C:\)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));

cfg->clearSearchPaths();
cfg->addSearchPath(R"(C:\$SHOT)");
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
#endif
}

Expand All @@ -181,7 +182,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
cs->setTransform(ft, OCIO::COLORSPACE_DIR_TO_REFERENCE);
cfg->addColorSpace(cs);

OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable());
OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable(minimal));

cfg->removeColorSpace("csTest");
};
Expand Down

0 comments on commit 7f1a112

Please sign in to comment.