template_pipeline.xml

<?xml version="1.0" encoding="UTF-8" ?>
<pipeline-def name="event-consolidation" description="This is the process for transforming event data" version="1.0.0">
    <settings>
        <singleSparkSession setting="true" />
        <globalViewAsLocal setting="true" />
    </settings>

    <aliases include="./src/test/resources/pipelines/miscellaneous/alias.xml">
        <alias name="setting" type="com.qwshen.etl.common.SparkConfActor" />
    </aliases>

    <udf-registration include="./src/test/resources/pipelines/miscellaneous/udf-registration.xml">
        <register prefix="user_" type="com.qwshen.etl.test.udf.UserUdf" />
    </udf-registration>

    <variables>
        <variable name="iam_password" value="${events.db.password}" decryptionKeyString="${application.security.decryption.key}" />
        <variable name="process_date" value="${application.process_date}" />
        <variable name="staging_uri" value="file:///c:/temp/staging" />
        <variable name="metrics_uri" value="file:///c:/temp/metrics" />
    </variables>

    <job include="./src/test/resources/pipelines/jobs/job.xml" />
    <job name="transform-user-events">
        <action name="load-users">
            <actor type="file">
                <properties>
                    <format>csv</format>
                    <options>
                        <header>false</header>
                        <delimiter>,</delimiter>
                        <quote>"</quote>
                        <timestampFormat>yyyy/MM/dd HH:mm:ss</timestampFormat>
                    </options>
                    <ddlSchemaString>user_id long, birth_year int, gender string, location string</ddlSchemaString>
                    <fileUri>${events.users_input}</fileUri>
                </properties>
            </actor>
            <output-view name="users" global="true" />
        </action>
        <action name="load-events">
            <actor type="flat">
                <properties>
                    <fileUri>${events.events_input}</fileUri>
                </properties>
            </actor>
            <output-view name="events_raw" global="false" />
        </action>
        <action name="transform-events">
            <actor type="sql">
                <properties>
                    <sqlString>
                        select
                            substr(row_value, 1, 12) as event_id,
                            substr(row_value, 13, 16) as event_time,
                            substr(row_value, 29, 12) as event_host,
                            substr(row_value, 41, 64) as event_location
                        from events_raw
                        where row_no not in (1, 2) and substr(row_value, 6, 5) != 'TFYKR'
                    </sqlString>
                </properties>
            </actor>
            <input-views>
                <view name="events_raw" />
            </input-views>
            <output-view name="events" global="true" />
        </action>
    </job>

    <metrics-logging enabled="true">
        <uri>${metrics_uri}</uri>
        <actions>
            <action name="load-events" />
        </actions>
    </metrics-logging>

    <debug-staging enabled="true">
        <uri>${staging_uri}</uri>
        <actions>
            <action name="transform-events" />
            <action name="load-events" />
        </actions>
    </debug-staging>
</pipeline-def>