Skip to content

Commit

Permalink
Merge branch '3.0.2'
Browse files Browse the repository at this point in the history
  • Loading branch information
dhutchis committed Oct 13, 2017
2 parents ee0b147 + 853340c commit 5082e8b
Show file tree
Hide file tree
Showing 68 changed files with 2,493 additions and 549 deletions.
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,8 @@ libext/
# Simulink Code Generation
slprj/

# Ignore Mac DS_Store files
.DS_Store

# Ignore *~ files
*~
2 changes: 1 addition & 1 deletion README.txt
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ U.S. Gov't Contractor, and Commercial. See additional documentation in the dist

Extract d4m_api.X.X.X.zip in your local directory.
If you want to connect to a database, then also download and extract the external libraries
libext.X.X.X.zip file and place it in the d4m_api/ directroy. This should result
libext.X.X.X.zip file and place it in the d4m_api/ directory. This should result
in a distribution containing:

d4m_api-X.X.X
Expand Down
7 changes: 3 additions & 4 deletions TEST/DBsetup.m
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
if not(exist('DB', 'var'))
global DB;

% Create a DB.
%DB = DBserver('classdb01.cloud.llgrid.ll.mit.edu:2181','Accumulo','accumulo',user,password);
%DB = DBsetupLLGrid('classdb51','/usr/local/tools');
DB = DBserver('localhost:2181','Accumulo','instance','root','secret');
% Create a DB.
DB = DBsetupLLGrid('class-db05');
%DB = DBserver('localhost:2181','Accumulo','instance','root','secret');
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Expand Down
6 changes: 3 additions & 3 deletions TEST/DBtablePairSecurityTEST.m
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@

catch e
deleteForce(T);
throw(e);
rethrow(e);
end

% Check Results
if(not(isempty(Tbar)))
throw(MException('ResultChk:SecurityError', '"foo" data should be inaccessable to "bar" queries'))
rethrow(MException('ResultChk:SecurityError', '"foo" data should be inaccessable to "bar" queries'))
end

if(isempty(Tfoo))
throw(MException('ResultChk:DataInaccessable', '"foo" data should be accessable to "foo" queries'))
rethrow(MException('ResultChk:DataInaccessable', '"foo" data should be accessable to "foo" queries'))
end

save([mfilename '.mat'],'-v6','Tfoo', 'Tbar');
Expand Down
6 changes: 3 additions & 3 deletions TEST/DBtableSecurityTEST.m
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@

catch e
deleteForce(T);
throw(e);
rethrow(e);
end

% Check Results
if(not(isempty(Tbar)))
throw(MException('ResultChk:SecurityError', '"foo" data should be inaccessable to "bar" queries'))
rethrow(MException('ResultChk:SecurityError', '"foo" data should be inaccessable to "bar" queries'))
end

if(isempty(Tfoo))
throw(MException('ResultChk:DataInaccessable', '"foo" data should be accessable to "foo" queries'))
rethrow(MException('ResultChk:DataInaccessable', '"foo" data should be accessable to "foo" queries'))
end

save([mfilename '.mat'],'-v6','Tfoo', 'Tbar');
Expand Down
6 changes: 4 additions & 2 deletions TEST/Reuters3MHtracks.m
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@
dMinLocDiff = zeros(1,dNl);

for iLoc = 1:dNl
dELpos =str2num(dEntLocPosMat(iLoc,:));
numLoc = find(isstrprop(dEntLocPosMat(iLoc,:), 'digit'),1,'last');
dELpos =str2num(dEntLocPosMat(iLoc,1:numLoc));
dNel = numel(dELpos);
dMinLocDiff(iLoc) = min(repmat(dELpos,[1 dNp]) ...
- reshape(repmat(dEntPerPosNum,[dNel 1]),[1 dNp*dNel]));
Expand All @@ -67,7 +68,8 @@


for iTime = 1:dNt
dETpos = str2num(dEntTimePosMat(iTime,:));
numLoc = find(isstrprop(dEntTimePosMat(iTime,:), 'digit'),1,'last');
dETpos = str2num(dEntTimePosMat(iTime,1:numLoc));
dNet = numel(dETpos);
dMinTimeDiff(iTime) = min( repmat(dETpos,[1 dNp]) ...
- reshape(repmat(dEntPerPosNum,[dNet 1]),[1 dNp*dNet]));
Expand Down
40 changes: 40 additions & 0 deletions TEST/scidbImportTest.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@

im = imread('peppers.png');
[nr, nc, ns] = size(im);

[ir, ic] = ind2sub([nr nc], 1:nr*nc);
ir = ir(:);
ic = ic(:);
slice = ones(size(ir));

dbname = 'txg-scidb01';
DB = DBsetupSciDB(dbname);

rowChunkSize = nr;
colChunkSize = nc;
sliceChunkSize = 1;

ls(DB)

T = DB( sprintf('D4Mtest_%s <val:uint8> [rows=1:%d,%d,0, cols=1:%d,%d,0, slice=1:%d,1,0]', datestr(now, 30), nr, nr, nc, nc, ns) );

% import data
putTriple( T, [ir ic slice], double( reshape(im(:,:,1), [nr*nc 1] ) ) );
putTriple( T, [ir ic slice*2], double( reshape(im(:,:,2), [nr*nc 1] ) ) );
putTriple( T, [ir ic slice*3], double( reshape(im(:,:,3), [nr*nc 1] ) ) );

% extract data
v1 = T(:,:,1);
disp( nnz( v1 - double(im(:,:,1)) ) ); % this should be 0

v2 = T(:,:,2);
disp( nnz( v2 - double(im(:,:,2)) ) ); % this should be 0

v3 = T(:,:,3);
disp( nnz( v3 - double(im(:,:,3)) ) ); % this should be 0

% cleanup
deleteForce(T);

ls(DB)

6 changes: 6 additions & 0 deletions VERSION.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,9 @@ SW-Version, Change, BuildDate...
2.5.0, upgrade to accumulo-1.5.0, Dec. 2 2013
2.5.1, fixes to make compatible with octave, Jan. 20 2014
2.5.3, fix visibility;add tablet server name for splits, Mar. 20 2014
2.6.1, upgrade to accumulo-1.6.1, Feb. 25 2015
2.6.2, adding graphulo, Sep. 24 2015
2.6.3, adding SciDB functionality, Aug. 30 2016
3.0.0, adding new graphulo functionality, upgrade to accumulo-1.8,faster ingest rates, Feb. 27, 2017
3.0.1, adding graphulo tests, improved minus, Mar. 31, 2017
3.0.2, adding larger/scaling graphulo tests, June 8, 2017
148 changes: 129 additions & 19 deletions build.xml
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,38 +1,148 @@
<?xml version="1.0"?>

<project name="d4m_api" default="all" basedir=".">
<property file="${basedir}/default_build.properties"/>
<!-- <property file="${basedir}/default_build.properties"/> -->

<import file="${basedir}/../d4m_api_java/build_common_targets.xml"/>

<target name="init_local" depends="init">
<filter filtersfile="${base_property_file_name}"/>
<filter filtersfile="${full_property_file_name}"/>
<filter token="version_number" value="${version_number}"/>

<ant antfile="${basedir}/../d4m_api_java/build.xml" dir="../d4m_api_java"
inheritAll="false" inheritRefs="false" target="build_jar" />
<copy todir="${basedir}/lib" file="${basedir}/../d4m_api_java/build/dist/lib/D4M_API_JAVA.jar"/>
<!-- <import file="${basedir}/../d4m_api_java/build_common_targets.xml"/> -->
<property name="build.dir" value="${basedir}/build"/>
<property name="lib.dir" value="${basedir}/lib"/>
<property name="src.dir" value="${basedir}/src"/>
<property name="test_src.dir" value="${basedir}/test"/>
<property name="dist.dir" value="${build.dir}/dist"/>
<property name="bin.dir" value="${build.dir}/dist/bin"/>
<property name="test.reports" value="${basedir}/build/reports"/>
<property name="version.num" value="3.0.0"/>

<target name="clean">
<delete dir="${build.dir}"/>
</target>

<target name="init_local" depends="init">
<copy todir="${dist.dir}">
<fileset dir="${basedir}">
<include name="docs/**"/>
<exclude name="docs/CHANGELOG.txt"/>
<include name="matlab_src/**"/>
<exclude name="matlab_src/html/**"/>
<include name="TEST/**"/>
<include name="lib/**"/>
<include name="libext/**"/>
<include name="examples/**"/>
<include name="COPYING.txt"/>
<include name="COPYRIGHT.txt"/>
<include name="README.txt"/>
<include name="VERSION.txt"/>
</fileset>
</copy>
</target>
<target name="init_test_release" depends="init">
<copy todir="${dist.dir}">
<fileset dir="${basedir}">
<include name="docs/**"/>
<include name="matlab_src/**"/>
<include name="TEST/**"/>
<include name="examples/**"/>
</fileset>
</copy>
<copy todir="${dist.dir}">
<fileset dir="${basedir}/../d4m_api_java/target">
<include name="graphulo-${version.num}-libext.zip"/>
</fileset>
</copy>
<copy todir="${dist.dir}/lib">
<fileset dir="${basedir}/../d4m_api_java/target">
<include name="graphulo-${version.num}.jar"/>
<include name="graphulo-${version.num}-alldeps.jar"/>
</fileset>
</copy>
<!--
<copy todir="${dist.dir}/matlab_src">
<fileset dir="${basedir}/../d4m_api_java/target">
<include name="DBinit.m"/>
</fileset>
</copy>
-->
</target>

<target name="init" depends="">
<mkdir dir="${build.dir}/dist"/>
<mkdir dir="${build.dir}/dist/docs"/>
<mkdir dir="${build.dir}/dist/examples"/>
<mkdir dir="${build.dir}/dist/lib"/>
<mkdir dir="${build.dir}/dist/libext"/>
</target>

<!-- copy scripts into bin directory -->
<copy todir="${dist.dir}/bin" >
<fileset dir="${basedir}/scripts">
<include name="**"/>
</fileset>
</copy>
<chmod dir="${dist.dir}/bin" perm="uag+x" includes="**/*.sh"/>
<target name="build_distribution" depends="">
<zip destfile="${build.dir}/${ant.project.name}_${version.num}.zip">
<zipfileset dir="${build.dir}/dist" prefix="${ant.project.name}_${version.num}">
<include name="**/**"/>
<exclude name="**/*.sh"/>
<exclude name="**/ant-contrib*.jar"/>
</zipfileset>
</zip>
</target>

<target name="all" depends="clean, init_local, build_distribution" description="Build a distribution of the whole software package"/>
<taskdef resource="net/sf/antcontrib/antcontrib.properties">
<classpath>
<pathelement location="${basedir}/lib/ant-contrib-1.0b3.jar"/>
</classpath>
</taskdef>

<target name="get-graphulo-libext-version">
<path id="artifact.id.path">
<fileset dir=".">
<include name="graphulo-*-libext.zip"/>
</fileset>
</path>
<property name="artifact.id.file" refid="artifact.id.path"/>
<echo message="artifact.id.file: ${artifact.id.file}"/>
<propertyregex property="graphulo.version" input="${artifact.id.file}" regexp=".*graphulo-(.*)-libext.zip" select="\1" />
<echo message="artifact.id: ${graphulo.version}"/>
</target>
<target name="unzip-libext" depends="init,get-graphulo-libext-version">
<unzip src="graphulo-${graphulo.version}-libext.zip" dest="${build.dir}/dist">
<patternset>
<include name="**/*.jar"/>
<exclude name="**/DBinit.m"/>
</patternset>
</unzip>

</target>

<target name="all"
depends="clean, init_local, unzip-libext, build_distribution"
description="Build a distribution of the whole software package"/>

<target name="my-echo" >
<fileset id="myfileset" dir="${basedir}/matlab_src">
<include name="*.m" />
</fileset>

<pathconvert pathsep="${line.separator}" property="sounds" refid="myfileset">
<!-- Add this if you want the path stripped -->
<mapper>
<flattenmapper />
</mapper>
</pathconvert>
<echo file="${basedir}/my_matlab_src.txt">${sounds}</echo>
</target>
<target name="make-test-release" description="Target for making test release using a snapshot of d4m_api_java"
depends="clean,init_test_release">
<!-- new lib -->
<!-- new libext -->
<!-- new DBinit.m -->
<unzip src="${dist.dir}/graphulo-${version.num}-libext.zip" dest="${build.dir}/dist">
<patternset>
<include name="**/*.jar"/>
<include name="**/DBinit.m"/>
</patternset>
</unzip>
<copy todir="${dist.dir}/matlab_src" overwrite="true" verbose="true">
<fileset dir="${dist.dir}">
<include name="DBinit.m"/>
</fileset>
</copy>
<delete file="${dist.dir}/DBinit.m"/>
<antcall target="build_distribution"/>
</target>
</project>
4 changes: 3 additions & 1 deletion docs/CHANGELOG.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ September 24, 2012
- Method for splitting tables
- New examples
- More documentation

- New helper function for SciDB : DBSetupScidb
- New functionality for inserting volumetric data into SciDB
- New functionality for retrieving 3D volumes from SciDB : v = T(:,:,:);

===================================================
May 16, 2012
Expand Down
34 changes: 34 additions & 0 deletions examples/1Intro/4Graphulo/DBsetup.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Setup binding to a database.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

myName = 'mytable_'; % SET LOCAL LABEL TO AVOID COLLISIONS.

[DB,G] = DBsetupLLGrid('class-db05'); % Create binding to database. Shorthand for:
%INSTANCENAME = 'instance-1.7.0';
%DB = DBserver('localhost:2181','Accumulo',INSTANCENAME,'root','secret');
%G = DBaddJavaOps('edu.mit.ll.graphulo.MatlabGraphulo',INSTANCENAME,'localhost:2181','root','secret');

% Create Adj Tables
TadjName=[myName 'Tadj'];
Tadj = DB(TadjName,[TadjName 'T']); % Create database table pair for holding adjacency matrix.
TadjDeg = DB([myName 'TadjDeg']); % Create database table for counting degree.

% Create Incidence Tables
TedgeName=[myName 'Tedge'];
Tedge = DB(TedgeName,[TedgeName 'T']); % Create database table pair for holding incidense matrix.
TedgeDeg = DB([myName 'TedgeDeg']); % Create database table for counting degree.

% Create Single Table
TsingleName=[myName 'Tsingle'];
Tsingle = DB(TsingleName); % Create database table pair for holding single table matrix.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% D4M: Dynamic Distributed Dimensional Data Model
% Architect: Dr. Jeremy Kepner (kepner@ll.mit.edu)
% Software Engineer: Dr. Jeremy Kepner (kepner@ll.mit.edu)
% MIT Lincoln Laboratory
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% (c) <2010> Massachusetts Institute of Technology
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Loading

0 comments on commit 5082e8b

Please sign in to comment.