diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..722d5e7
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+.vscode
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..a9fe8d2
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,22 @@
+
+The MIT License (MIT)
+
+Copyright (c) Daniel Wellington AB 2018
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..2622d26
--- /dev/null
+++ b/README.md
@@ -0,0 +1,336 @@
+# StepTest
+
+Package and program made to make transactional load test easy.
+
+## Motivation
+
+Most other load testing packages/frameworks was made for testing REST APIs in a non transactional way.
+This package was born out of the necessity to test our Magento based eCommerce platforms checkout over
+multiple website ids and payment methods.
+
+For this we needed a package that can, in a flexible way, test multiple steps in a transactional manner.
+It must also be able to loop over values and also set variables based on the result body / headers from
+previous transaction.
+
+It also has the possibility to replay real scenarios. Jobs can be added with a "start after" value.
+Making it possible to replay old load exactly as it happened.
+
+## Usage
+
+The first part of making StepTest is work is defining a steps -file.
+This includes the different steps that the Server will run for each job that is added with the specified steps -file.
+
+The steps -files syntax support a range of different functions such as `VAR`, `VARFROM`, `ARRAY`, `FOR`, `AUTH`, `HEADER`, `COOKIE` and of course HTTP
+functions such as `GET`, `POST`, `PUT`, `PATCH`, `DELETE`. Functions can be declared either in upper or lower case.
+
+Each step is divided by a dash `-`, any leading/trailing spaces and tabs will be removed.
+
+## Example
+
+### steps.txt
+
+ - var { "name": "url", "value": "example.com" }
+ array { "name": "productList", "values": [ "prodId1", "prodId2", "prodId3" ] }
+
+ - get https://{{url}}/getSession
+ varfrom { "from": "body", "name": "session", "syntax": ""}
+
+ - for product in {{productList}}
+ post https://{{url}}/addProduct {"session":"{{session}}","product":"{{product}}"}
+ forend
+
+ - get https://{{url}}/getCart
+
+### main.go
+
+```go
+package main
+
+import (
+ "fmt"
+ "log"
+
+ "github.com/dwtechnologies/steptest"
+)
+
+func main() {
+ // Create a new StepTest server with 10 virtual users and 15s http timeout.
+ srv, err := steptest.New(10, 15)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // Add a job with no vars or startAfter value.
+ srv.AddJob("steps.txt", 1, nil, nil)
+
+ // Start the server and then wait until StepTest has finished all requests.
+ srv.Start()
+ srv.WaitDone()
+
+ // Print some results.
+ if errors := srv.GetNumberOfErrors(); errors > 0 {
+ fmt.Printf(">>> Number of errors: %d\n\n", errors)
+
+ for _, err := range srv.GetErrorMessages() {
+ fmt.Printf("%s\n", err.Error)
+ }
+ fmt.Printf("\n")
+ }
+
+ fmt.Printf(">>> Total number of fetches: %d\n", srv.GetNumberOfRequests())
+ fmt.Printf(">>> Average time: %d ms\n", srv.GetAverageFetchTime())
+ fmt.Printf(">>> Total runtime: %d s\n", srv.GetTotalRunTime())
+}
+```
+
+## Reference - Stepfile
+
+Every line that starts with a dash followed by a space will be defined as a step separator.
+Every function in a step is divided by every line that starts with two spaces.
+
+### GET
+
+`get http://example.com`
+
+> Creates a new GET request against http://example.com
+
+### POST
+
+`post http://example.com {"name":"value"}`
+
+> Creates a new POST request against http://example.com with a JSON body.
+
+### PUT
+
+`put http://example.com name%3Dvalue`
+
+> Creates a new PUT request against http://example.com with a URL Encoded body.
+
+### PATCH
+
+`patch http://example.com/id/1234 {"partial":"info"}`
+
+> Creates a new PATCH request against http://example.com with a JSON body.
+
+### DELETE
+
+`delete http://example.com/id/1234`
+
+> Creates a new DELETE request against http://example.com with a empty body.
+
+### VAR
+
+`var { "name": "var1", "value": "val1" }`
+
+> Creates a new variable called var1 with a value of val1.
+
+### ARRAY
+
+`array { "name": "arr1", "values": [ "val1", "val2", "val3" ] }`
+
+> Creates a new array called arr1 with values val1, val2 and val3.
+
+### VARFROM
+
+`varfrom { "from": "body", "name": "var1", "syntax": "" }`
+
+> Creates a variable called var1. The value of var1 will be based on the requests BODY where it will look for the syntax ``. And anything thats contained in the `{{StepTestSyntax}}` will be the value of the variable.
+
+### COOKIE
+
+`cookie { }`
+
+> Creates a new cookie with the values...
+
+### HEADER
+
+`header { "name": "header1", "value": "val1" }`
+
+> Creates a new header with name header1 and value val1. (local to the step)
+
+### \@HEADER
+
+`@header { "name": "header1", "value": "val1" }`
+
+> Creates a new global header with name header1 and value val1. (global for whole job)
+
+### AUTH
+
+`auth { "username": "user1", "password": "pass1" }`
+
+> Adds Auth to the request with username and password user1 and pass1. (local to the step)
+
+### \@AUTH
+
+`@auth { "username": "user1", "password": "pass1" }`
+
+> Adds Global Auth to the request with username and password user1 and pass1. (global for whole job)
+
+### FOR
+
+`for i in {{arr1}}`
+`for i in [ "val1", "val2", "val3" ]`
+`for i in {{var1}}` // var1 needs to contain a stringified JSON array that can be unmarshaled.
+
+> Creates a for loop that will loop through all the values in the array and set the variable i to the value
+> from the array. More than one step can be included in the forloop. Should be ended with a forend.
+> The step that the for is defined in will be included in the for loop.
+
+### FOREND
+
+`forend`
+
+> Ends a for loop. Can be part of the same step as for. Then only that step will be looped over.
+
+## Reference - Exported functions
+
+### New
+
+```go
+steptest.New(v int, t int) (*Server, error)
+```
+
+> New takes a number of virtual users v and request timeout t and creates a StepTest Server.
+> Returns *Server and error.
+
+### AddJob
+
+```go
+*Server.AddJob(s string, r int, v map[string]string, a *time.Time) error
+```
+
+> AddJob will parse a job and add it to the *Server.
+> It takes the path to a stepsfile s, r number of runs, v variables as a map of strings
+> and time a when to start the job, for direct execution just nil.
+> Returns error.
+
+### Start
+
+```go
+*Server.Start()
+```
+
+> Start will start the execution of parsed jobs. If there are any unparsed jobs left
+> in the queue it will wait for them to finish before starting execution.
+
+### StopParsing
+
+```go
+*Server.StopParsing() error
+```
+
+> StopParsing will send a signal to stop all parsing being done on the Server.
+> StopParsing can only be called when the Server is in a IsParsing -state.
+> Returns error.
+
+### StopRunning
+
+```go
+*Server.StopRunning() error
+```
+
+> StopRunning will send a signal to stop fetching requests on the Server.
+> StopRunning can only be called when the Server is in a IsRunning -state.
+> Returns error.
+
+### WaitDone
+
+```go
+*Server.WaitDone()
+```
+
+> WaitDone will wait until the Server has finished fetching all the requests in the *Server.jobs map.
+> WaitDone will block the program until it has finished.
+
+### GetNumberOfVirtualUsers
+
+```go
+*Server.GetNumberOfVirtualUsers() int
+```
+
+> GetNumberOfVirtualUsers returns the number of virtual users.
+> Returns int.
+
+### GetNumberOfJobs
+
+```go
+*Server.GetNumberOfJobs() int
+```
+
+> GetNumberOfJobs returns the number of jobs stored on the Server.
+> Returns int
+
+### GetNumberOfRequests
+
+```go
+*Server.GetNumberOfRequests() int
+```
+
+> GetNumberOfRequests returns the number of successfull requests.
+> Returns int.
+
+### GetNumberOfErrors
+
+```go
+*Server.GetNumberOfErrors() int
+```
+
+> GetNumberOfErrors will return the amount of requests that errored.
+> Returns int.
+
+### GetErrorMessages
+
+```go
+*Server.GetErrorMessages() []error
+```
+
+> GetErrorMessages will return all the error messages since the Server was started.
+> Returns Error.
+
+### GetAverageFetchTime
+
+```go
+*Server.GetAverageFetchTime() time.Duration
+```
+
+> GetAverageFetchTime will return the average fetch time for all the requests. Requests that resultet in errors will be ignored in the average.
+> Returns time.Duration.
+
+### IsParsing
+
+```go
+*Server.IsParsing() bool
+```
+
+> IsParsing returns true if the Server is still parsing jobs. False if it has finished or manually been stopped.
+> Returns bool.
+
+### IsRunning
+
+```go
+*Server.IsRunning() bool
+```
+
+> IsRunning returns true if the Server is still running jobs. False if it has finished or manually been stopped.
+> Returns bool.
+
+### GetTotalRunTime
+
+```go
+*Server.GetTotalRunTime() time.Duration
+```
+
+> GetTotalRunTime will return the total runtime since Server start.
+> Returns time.Duration.
+
+## Installation
+
+`go get -u github.com/dwtechnologies/steptest`
+
+## Contributors
+
+To improve on the project, please submit a pull request.
+
+## License
+
+The code is copyright under the MIT license.
\ No newline at end of file
diff --git a/add_options.go b/add_options.go
new file mode 100644
index 0000000..0219bb7
--- /dev/null
+++ b/add_options.go
@@ -0,0 +1,76 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "net/http"
+)
+
+// addOptions adds Headers, Cookies and Basic Auth from step s to request req.
+func (j *job) addOptions(s *step, req *http.Request) {
+ j.setUserAgent(req)
+ j.addBasicAuth(s, req)
+ j.addHeaders(s, req)
+ j.addCookies(s, req)
+}
+
+// setUserAgent will set the User Agent of the request o Chrome Chrome/64.0.3282.186 on Mac OS X 10.13.3
+func (*job) setUserAgent(req *http.Request) {
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36")
+}
+
+// addBasicAuth adds either local or global Basic Auth from step s to *http.Request req.
+// If both local and global auth parameters are set the local will take precedence.
+func (j *job) addBasicAuth(s *step, req *http.Request) {
+ switch {
+ case s.auth.Username != "" && s.auth.Password != "":
+ req.SetBasicAuth(s.auth.Username, s.auth.Password)
+
+ case j.globalAuth.Username != "" && j.globalAuth.Password != "":
+ req.SetBasicAuth(j.globalAuth.Username, j.globalAuth.Password)
+ }
+}
+
+// addHeaders sets headers from step s to *http.Request req.
+// If header is already set it will be overwritten with the new value.
+func (*job) addHeaders(s *step, req *http.Request) {
+ for _, header := range s.headers {
+ req.Header.Set(header.Name, header.Value)
+ }
+}
+
+// addCookies adds cookies from step s to *http.Request req.
+// If cookie is already set it will be overwritten with the new value.
+func (*job) addCookies(s *step, req *http.Request) {
+ for _, cookie := range s.cookies {
+ req.AddCookie(&cookie)
+ }
+}
+
+// appendResponseCookiesToJob will append/replace the cookies that we received from the response res to job j.
+// So that cookies received will automatically be added to the next step of the job.
+func (j *job) appendResponseCookiesToJob(res *http.Response) {
+ for _, cookie := range res.Cookies() {
+ exists := false
+
+ // If we already have the cookie no need to add it again. Just update the value.
+ for i := range j.cookies {
+ if exists = j.compareCookie(cookie, &j.cookies[i]); exists {
+ j.cookies[i].Value = cookie.Value
+ }
+ }
+
+ if !exists {
+ j.cookies = append(j.cookies, *cookie)
+ }
+ }
+}
+
+// compareCookie will check if name, domain, path are the same for *http.Cookie c1 and c2.
+// If they are the same the returned value will be true.
+// Returns bool.
+func (*job) compareCookie(c1 *http.Cookie, c2 *http.Cookie) bool {
+ if c1.Name == c2.Name && c1.Domain == c2.Domain && c1.Path == c2.Path {
+ return true
+ }
+ return false
+}
diff --git a/check_conditions.go b/check_conditions.go
new file mode 100644
index 0000000..a0aade3
--- /dev/null
+++ b/check_conditions.go
@@ -0,0 +1,21 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+// checkConditions will check the steps if/conditions and return true if any of the conditions matched.
+// Returns boolean.
+func (j *job) checkConditions(s *step) bool {
+ if len(s.conditions) == 0 {
+ return true
+ }
+
+ for _, c := range s.conditions {
+ switch c.Type {
+ case "exists":
+ if _, ok := j.vars[c.Var1]; ok {
+ return true
+ }
+ }
+ }
+
+ return false
+}
diff --git a/jobs_add.go b/jobs_add.go
new file mode 100644
index 0000000..ccf83a5
--- /dev/null
+++ b/jobs_add.go
@@ -0,0 +1,44 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "fmt"
+)
+
+// AddJob will parse a job and add it to the *Server.
+// It takes steps s, v variables as a map of strings,
+// time a when to start the job. Time a can be nil for direct/orderless execution.
+// Returns error.
+func (srv *Server) AddJob(s string, v map[string]string) error {
+ if srv.parsedJobs == nil {
+ return fmt.Errorf("Error adding job in *Server.AddJob. *Server.parsedJobs channel is closed")
+ }
+
+ j, err := srv.parseJob(&rawJob{s, v})
+ if err != nil {
+ return err
+ }
+ go srv.workerAddJob(j)
+
+ return nil
+}
+
+// workerAddJob will send the job j to the *Server.parsedJob channel.
+// Since this channel is unbuffered we need to run this function in a separate go-routine.
+// We need to deep copy all maps and/or slices in each iteration due to the nature of how
+// these are passed (by reference) by go.
+func (srv *Server) workerAddJob(j *job) {
+ srv.wgRun.Add(1)
+ srv.parsedJobs <- j
+ srv.addedJobsCounterChan <- 1
+}
+
+// workerAddedJobCounter will listen for incoming increment changes on the
+// *Server.addedJobsCounterChan channel and add those to the *Server.addedJobsCounter.
+// This way we can track the amount of added jobs.
+func (srv *Server) workerAddedJobCounter() {
+ for {
+ inc := <-srv.addedJobsCounterChan
+ srv.addedJobsCounter += inc
+ }
+}
diff --git a/jobs_add_test.go b/jobs_add_test.go
new file mode 100644
index 0000000..17384a3
--- /dev/null
+++ b/jobs_add_test.go
@@ -0,0 +1,60 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "testing"
+)
+
+func TestAddJob(t *testing.T) {
+ // Create the server.
+ srv, err := New(10, 30000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ steps := "- GET https://{{url}}\n"
+ steps += ` var { "name": "url", "value": "google.com" }`
+ steps += "\n\n"
+ steps += "- POST https://www.sunet.se/"
+
+ err = srv.AddJob(steps, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ // Receive jobs.
+ jobs := []*job{}
+ for i := 0; i < 5; i++ {
+ job := <-srv.parsedJobs
+ jobs = append(jobs, job)
+ <-srv.addedJobsCounterChan
+ }
+
+ // Check that the added job has been added to run 5 times.
+ if len(jobs) != 5 {
+ t.Errorf("Wrong amount of runs for the specified job seems to have added. Expected %d but got %d", 5, len(jobs))
+ }
+
+ // Check that the data for each job is correct.
+ for _, job := range jobs {
+ if job.vars["url"] != "google.com" {
+ t.Errorf("Expected var url to be %s but got %s", "google.com", job.vars["url"])
+ }
+
+ // Check first step (google)
+ if job.steps[0].method != "GET" {
+ t.Errorf("Expected first METHOD of steps to be %s but got %s", "GET", job.steps[0].method)
+ }
+ if job.steps[0].url != "https://{{url}}" {
+ t.Errorf("Expected first URL of steps to be %s but got %s", "https://{{url}}", job.steps[0].url)
+ }
+
+ // Check second step (sunet.se)
+ if job.steps[1].method != "POST" {
+ t.Errorf("Expected second METHOD of steps to be %s but got %s", "POST", job.steps[0].method)
+ }
+ if job.steps[1].url != "https://www.sunet.se/" {
+ t.Errorf("Expected second URL of steps to be %s but got %s", "https://www.sunet.se/", job.steps[0].url)
+ }
+ }
+}
diff --git a/jobs_parse.go b/jobs_parse.go
new file mode 100644
index 0000000..0d2f3b8
--- /dev/null
+++ b/jobs_parse.go
@@ -0,0 +1,530 @@
+// Package steptest makes transactional load test easy.
+//FIXME: Whole jobs_parse.go should be replaced by something fancier and more javascript-syntax like.
+//And support for nested for loops, scoped variables and such. Next version :) :) :)
+package steptest
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+)
+
+const (
+ stepSeparatorRegexp = "(?m:^- )" // Each step is divided by a leading dash and a following space.
+ lineSeparatorRegexp = "(?m:^ )" // Each function in a step is divided by two leading spaces.
+ removeVarCurls = "(?m:^{{|}}$)" // Regexp to remove curls from brackets.
+ separator = " " // Each command, value etc. is separated by a space.
+ emptyRow = "" // Empty rows will be blank, since we trim whitespaces.
+ trim = " \t" // Trim whitespaces and tabs.
+ newline = "\n" // Character to match newlines.
+ forInSeparator = "in" // Separator between variable name and array.
+)
+
+var (
+ // Allowed condition types for the if/condition statement.
+ allowedConditions = []string{"exists", "equals", "greater", "less", "true", "false"}
+)
+
+// stepTypes contains all the supported functions of the stepsfile.
+// If any row begins with anything else than described below it will result in an error.
+// Variables and cookies are always global. So keep this in mind that there are no scopes.
+// Auth and headers can be either local to the step or global. Declare global auth headers
+// by adding @ in front of cookie/header.
+// Rows only containing one newline will be ignored.
+var stepTypes = map[string]func(*job, *step, *string) error{
+ "get": createGet,
+ "post": createPost,
+ "patch": createPatch,
+ "put": createPut,
+ "delete": createDelete,
+ "var": createVar,
+ "array": createArray,
+ "varfrom": createVarFrom,
+ "cookie": createCookie,
+ "header": createHeader,
+ "auth": createAuth,
+ "@header": createGlobalHeader,
+ "@auth": createGlobalAuth,
+ "for": startForLoop,
+ "forend": endForLoop,
+ "if": createIf,
+}
+
+// parseJob takes raw job r and creates a job out of it.
+// It then parses r.steps and turns it into a parsed job.
+// Returns *job and error.
+func (srv *Server) parseJob(r *rawJob) (*job, error) {
+ j := &job{arrays: make(map[string][]string), vars: r.vars}
+
+ if j.vars == nil {
+ j.vars = make(map[string]string)
+ }
+
+ err := j.createSteps(r)
+ if err != nil {
+ return nil, err
+ }
+
+ return j, nil
+}
+
+// createSteps reads the stepsfile s and splits it into steps based on the stepSeparator.
+// It will iterate over each step and call *job.createStep for each step.
+// Returns error.
+func (j *job) createSteps(r *rawJob) error {
+ regexp, err := regexp.Compile(stepSeparatorRegexp)
+ if err != nil {
+ return fmt.Errorf("Couldn't compile regular expression in *job.createSteps. %s", err.Error())
+ }
+
+ steps := regexp.Split(r.steps, -1)
+ for _, s := range steps {
+ // Skip empty steps only containing spaces / tabs or are empty.
+ if strings.Trim(s, trim) == "" {
+ continue
+ }
+
+ err := j.createStep(&s)
+ if err != nil {
+ return err
+ }
+ }
+
+ // If forcounter is greater or smaller than 0, we had a FOR
+ // loop without a FOREND or FOREND with without a FOR.
+ switch {
+ case j.forcounter > 0:
+ return fmt.Errorf("Received for statement without a forend in steps in *job.createSteps")
+ case j.forcounter < 0:
+ return fmt.Errorf("Received forend statement without a for in steps in *job.createSteps")
+ }
+
+ return nil
+}
+
+// createStep takes step s and splits it into lines based on the lineSeparator.
+// It will iterate over each line and call *job.createStepLine for each line.
+// Returns error.
+func (j *job) createStep(s *string) error {
+ stp := new(step)
+ regexp, err := regexp.Compile(lineSeparatorRegexp)
+ if err != nil {
+ return fmt.Errorf("Couldn't compile regular expression in *job.createStep. %s", err.Error())
+ }
+
+ // Remove all newlines and split by lineSeparatorRegexp.
+ r := regexp.Split(*s, -1)
+ for _, row := range r {
+ row = strings.Replace(row, newline, "", -1)
+ err := j.createStepLine(stp, &row)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Determine if we should add the step to the job or to a for loop.
+ // If the addTo slice is not empty, the tep should be added to a for loop step.
+ // Otherwise we will hit the default case, which is just to add it as a regular
+ // step directly on the jobs steps slice.
+ switch {
+ case len(j.addTo) > 0:
+ j.addStepToForLoop(stp)
+
+ default:
+ j.addStepToJob(stp)
+ }
+
+ // If we should leave the for loop for next step, remove the value form j.addTo.
+ // Since we don't support nested for loops at this time we just remove the slice
+ // otherwise we would pop the last value out.
+ if j.forRemoveNextStep {
+ j.forRemoveNextStep = false
+ j.addTo = []int{}
+ }
+
+ return nil
+}
+
+// addStepToJob will add a step to the global steps slice of the job. So all regular steps
+// will be added by this function.
+func (j *job) addStepToJob(stp *step) {
+ j.steps = append(j.steps, *stp)
+}
+
+// addStepToForLoop will add the step to the for loop steps slice. All steps belonging to a for loop
+// will be added here. Note nested for loops are not supported at this time!
+func (j *job) addStepToForLoop(stp *step) {
+ if len(j.steps) == j.addTo[0] {
+ j.steps = append(j.steps, step{forloop: stp.forloop})
+ }
+
+ stp.forloop = forloop{}
+ j.steps[j.addTo[0]].forloop.steps = append(j.steps[j.addTo[0]].forloop.steps, *stp)
+}
+
+// createStepLine will call the function based on what keyword is defined in the step.
+// See stepTypes for the different types/keywords. Any empty rows will be ignored.
+// We will trim all leading and empty spaces so that empty rows with a singel space will not cause an error.
+// We will assign the function to f based on the stepTypes map.
+// Returns error.
+func (j *job) createStepLine(step *step, r *string) error {
+ s := strings.SplitN(strings.Trim(*r, trim), separator, 2)
+ t := strings.ToLower(s[0])
+
+ if t == emptyRow {
+ return nil
+ }
+
+ f, ok := stepTypes[t]
+ if !ok {
+ return fmt.Errorf("Couldn't find function type %s in stepTypes map in *job.createStepLine", t)
+ }
+
+ args := ""
+ if len(s) > 1 {
+ args = s[1]
+ }
+
+ err := f(j, step, &args)
+ return err
+}
+
+// createGet will create a HTTP GET step based on step s and args a by
+// calling *step.createHTTPStep. job j is not needed and will be ignored.
+// Returns error.
+func createGet(j *job, s *step, a *string) error {
+ return s.createHTTPStep("GET", a)
+}
+
+// createGet will create a HTTP POST step based on step s and args a by
+// calling *step.createHTTPStep. job j is not needed and will be ignored.
+// Returns error.
+func createPost(j *job, s *step, a *string) error {
+ return s.createHTTPStep("POST", a)
+}
+
+// createGet will create a HTTP PATCH step based on step s and args a by
+// calling *step.createHTTPStep. job j is not needed and will be ignored.
+// Returns error.
+func createPatch(j *job, s *step, a *string) error {
+ return s.createHTTPStep("PATCH", a)
+}
+
+// createGet will create a HTTP PUT step based on step s and args a by
+// calling *step.createHTTPStep. job j is not needed and will be ignored.
+// Returns error.
+func createPut(j *job, s *step, a *string) error {
+ return s.createHTTPStep("PUT", a)
+}
+
+// createGet will create a HTTP DELETE step based on step s and args a by
+// calling *step.createHTTPStep. job j is not needed and will be ignored.
+// Returns error.
+func createDelete(j *job, s *step, a *string) error {
+ return s.createHTTPStep("DELETE", a)
+}
+
+// createHTTPStep will take m method and a args and set the correct method, url
+// and body to the step. If method is GET or no body argument is supplied no body will be set.
+// Returns error.
+func (s *step) createHTTPStep(m string, a *string) error {
+ v := strings.SplitN(*a, separator, 2)
+
+ switch {
+ case v[0] == "":
+ return fmt.Errorf("%s was declared but URL was not supplied in *step.createHTTPStep. Raw %s", m, *a)
+
+ case len(v) > 1 && m != "GET":
+ s.body = v[1]
+ }
+
+ s.method = m
+ s.url = v[0]
+ return nil
+}
+
+// createVar will add a variable from args a to the jobs j vars map. Step s will be ignored.
+// These variables will be global and accessible to the whole job after they have been declared.
+// Returns error.
+func createVar(j *job, s *step, a *string) error {
+ v := new(variable)
+ err := json.Unmarshal([]byte(*a), v)
+ if err != nil {
+ return fmt.Errorf("var was declared but we couldn't unmarshal it in createVar. Raw %s", *a)
+ }
+
+ switch {
+ case v.Name == "":
+ return fmt.Errorf("var was declared but NAME was not supplied in createVar. Raw %s", *a)
+
+ case v.Value == "":
+ return fmt.Errorf("var was declared but VALUE was not supplied in createVar. Raw %s", *a)
+ }
+
+ j.vars[v.Name] = v.Value
+ return nil
+}
+
+// createVarFrom will add a variable to the jobs j vars map depending on the result from the steps HTTP request.
+// The value can be fetched by specifying either BODY or HEADER and then specifying a pattern to look for in args a.
+// Substitute the value to get from the search syntax with searchSyntax. Step s will be ignored.
+// Returns error.
+func createVarFrom(j *job, s *step, a *string) error {
+ v := new(varfromItem)
+ err := json.Unmarshal([]byte(*a), v)
+ if err != nil {
+ return fmt.Errorf("varfrom was declared but we couldn't unmarshal it in createVarFrom. Raw %s", *a)
+ }
+
+ switch {
+ case v.From == "":
+ return fmt.Errorf("varfrom was declared but FROM was not supplied in createVarFrom. Raw %s", *a)
+
+ case v.Varname == "":
+ return fmt.Errorf("varfrom was declared but NAME was not supplied in createVarFrom. Raw %s", *a)
+
+ case v.OrgSyntax == "":
+ return fmt.Errorf("varfrom was declared but FIND was not supplied in createVarFrom. Raw %s", *a)
+ }
+
+ v.Syntax = *j.createSearchPattern(&v.OrgSyntax)
+ s.varfrom = append(s.varfrom, *v)
+ return nil
+}
+
+// createSearchPattern will take an input pattern p and convert it to a regular expression we can use to parse the BODY/HEADERS of a result.
+// It does this by replace the searchSyntax word with searchSyntaxReplace and then inserting it inside searchSyntaxRegexp.
+func (*job) createSearchPattern(p *string) *string {
+ n := fmt.Sprintf(searchSyntaxRegexp, strings.Replace(*p, searchSyntax, searchSyntaxReplace, -1))
+ return &n
+}
+
+// createCookie will append a cookie to the jobs j cookies slice based on the data in args a. Step s will be ignored.
+// Returns error.
+func createCookie(j *job, s *step, a *string) error {
+ c := new(cookie)
+ err := json.Unmarshal([]byte(*a), c)
+ if err != nil {
+ return fmt.Errorf("cookie was declared but we couldn't unmarshal it in createCookie. Raw %s", *a)
+ }
+
+ switch {
+ case c.Name == "":
+ return fmt.Errorf("cookie was declared but NAME was not supplied in createCookie. Raw %s", *a)
+
+ case c.Value == "":
+ return fmt.Errorf("cookie was declared but VALUE was not supplied in createCookie. Raw %s", *a)
+ }
+
+ j.cookies = append(j.cookies, http.Cookie{
+ Name: c.Name,
+ Value: c.Value,
+ Path: c.Path,
+ Domain: c.Domain,
+ Expires: c.Expires,
+ MaxAge: c.MaxAge,
+ Secure: c.Secure,
+ HttpOnly: c.HTTPOnly,
+ })
+ return nil
+}
+
+// createHeader will append a header to the steps headers slice based on the data in args a.
+// Headers added with createHeader will be local to the specified step s only. Jobs j will be ignored.
+// Returns error.
+func createHeader(j *job, s *step, a *string) error {
+ h := new(header)
+ err := json.Unmarshal([]byte(*a), h)
+ if err != nil {
+ return fmt.Errorf("header was declared but we couldn't unmarshal it in createHeader. Raw %s", *a)
+ }
+
+ switch {
+ case h.Name == "":
+ return fmt.Errorf("header was declared but NAME was not supplied in createHeader. Raw %s", *a)
+
+ case h.Value == "":
+ return fmt.Errorf("header was declared but VALUE was not supplied in createHeader. Raw %s", *a)
+ }
+
+ s.headers = append(s.headers, *h)
+ return nil
+}
+
+// createAuth will add Basic Auth to the step s based on data in args a.
+// Basic Auth added with createAuth will be local to the specified step s only. Jobs j will be ignored.
+// Returns error.
+func createAuth(j *job, s *step, a *string) error {
+ ba := new(auth)
+ err := json.Unmarshal([]byte(*a), ba)
+ if err != nil {
+ return fmt.Errorf("auth was declared but we couldn't unmarshal it in createAuth. Raw %s", *a)
+ }
+
+ switch {
+ case ba.Username == "":
+ return fmt.Errorf("auth was declared but USERNAME was not supplied in createAuth. Raw %s", *a)
+
+ case ba.Password == "":
+ return fmt.Errorf("auth was declared but PASSWORD was not supplied in createAuth. Raw %s", *a)
+ }
+
+ s.auth.Username, s.auth.Password = ba.Username, ba.Password
+ return nil
+}
+
+// createGlobalHeader will append a header to the jobs j headers slice based on the data in args a.
+// Headers added with createGlobalHeader will be global to all steps in job j. Step s will be ignored.
+// Returns error.
+func createGlobalHeader(j *job, s *step, a *string) error {
+ h := new(header)
+ err := json.Unmarshal([]byte(*a), h)
+ if err != nil {
+ return fmt.Errorf("@header was declared but we couldn't unmarshal it in createGlobalHeader. Raw %s", *a)
+ }
+
+ switch {
+ case h.Name == "":
+ return fmt.Errorf("@header was declared but NAME was not supplied in createGlobalHeader. Raw %s", *a)
+
+ case h.Value == "":
+ return fmt.Errorf("@header was declared but VALUE was not supplied in createGlobalHeader. Raw %s", *a)
+ }
+
+ j.globalHeaders = append(j.globalHeaders, *h)
+ return nil
+}
+
+// createGlobalAuth will add Basic Auth to the job j based on data in args a.
+// Basic Auth added with createGlobalAuth will be global to all steps in job j. Step s will be ignored.
+// Returns error.
+func createGlobalAuth(j *job, s *step, a *string) error {
+ ba := new(auth)
+ err := json.Unmarshal([]byte(*a), ba)
+ if err != nil {
+ return fmt.Errorf("@auth was declared but we couldn't unmarshal it in createGlobalAuth. Raw %s", *a)
+ }
+
+ switch {
+ case ba.Username == "":
+ return fmt.Errorf("@auth was declared but USERNAME was not supplied in createGlobalAuth. Raw %s", *a)
+
+ case ba.Password == "":
+ return fmt.Errorf("@auth was declared but PASSWORD was not supplied in createGlobalAuth. Raw %s", *a)
+ }
+
+ j.globalAuth.Username, j.globalAuth.Password = ba.Username, ba.Password
+ return nil
+}
+
+// startForLoop will create a for that will loop all the steps contained within based on the supplied separator.
+// It will run until len of the forloop struct becomes zero. And loop between the step for was declared and forend.
+// Returns error.
+func startForLoop(j *job, s *step, a *string) error {
+ f := strings.SplitN(*a, separator, 3)
+ switch {
+ case len(j.addTo) > 0:
+ return fmt.Errorf("Sorry, but nested FOR loops are not yet supported in startForLoop. Raw %s", *a)
+
+ case len(f) < 3:
+ return fmt.Errorf("for was declared but with an invalid syntax. FOR needs to be in 'for VARNAME in ARRAY' format in createFor. Raw %s", *a)
+
+ case strings.ToLower(f[1]) != forInSeparator:
+ return fmt.Errorf("for was declared but with an invalid syntax. FOR needs to be in 'for VARNAME in ARRAY' format in createFor. Raw %s", *a)
+ }
+
+ arr := new([]string)
+
+ // If we couldn't unmarshal the data but the stored data looks to be an variable/array.
+ // Search for the name of the variable in the jobs array list. If it exists we can safely
+ // set the values to that array. If it doesn't exists we return error.
+ regexp, err := regexp.Compile(removeVarCurls)
+ if err != nil {
+ return fmt.Errorf("Couldn't compile regular expression in *job.startForLoop. %s", err.Error())
+ }
+
+ _, ok := j.arrays[regexp.ReplaceAllString(f[2], "")]
+
+ switch {
+ case ok:
+ *arr = []string{f[2]}
+
+ default:
+ err := json.Unmarshal([]byte(f[2]), arr)
+ if err != nil {
+ return fmt.Errorf("for was declared but we couldn't unmarshal values in it in createFor. Raw %s", *a)
+ }
+ }
+
+ s.forloop = forloop{varname: f[0], values: *arr}
+
+ j.addTo = []int{len(j.steps)} // Add the current steps index to the addTo slice. Hardcorded for now...
+ j.forcounter++
+ return nil
+}
+
+// endForLoop will end a previously created for loop. If no previous for loop was declared it will return error.
+// Returns error.
+func endForLoop(j *job, s *step, a *string) error {
+ if j.forcounter < 1 {
+ return fmt.Errorf("forend was encountered but no for was declared previously")
+ }
+
+ j.forRemoveNextStep = true
+ j.forcounter--
+ return nil
+}
+
+// createArray creates an array that can be used by other functions such as rand.
+// Returns error.
+func createArray(j *job, s *step, a *string) error {
+ ar := new(array)
+ err := json.Unmarshal([]byte(*a), ar)
+ if err != nil {
+ return fmt.Errorf("array was declared but we couldn't unmarshal it in createArray. Raw %s", *a)
+ }
+
+ switch {
+ case ar.Name == "":
+ return fmt.Errorf("array was declared but NAME was not supplied in createArray. Raw %s", *a)
+
+ case len(ar.Values) == 0:
+ return fmt.Errorf("array was declared but VALUES was not supplied in createArray. Raw %s", *a)
+ }
+
+ j.arrays[ar.Name] = ar.Values
+ return nil
+}
+
+// createIf will create a conditional variable based on the supplied condition.
+// It will also check that the supplied condition type is valid.
+// Returns error.
+func createIf(j *job, s *step, a *string) error {
+ i := new(condition)
+ err := json.Unmarshal([]byte(*a), i)
+ if err != nil {
+ return fmt.Errorf("if was declared but we couldn't unmarshal it in createIf. Raw %s", *a)
+ }
+
+ switch {
+ case i.Type == "":
+ return fmt.Errorf("if was declared but TYPE was not supplied in createIf. Raw %s", *a)
+
+ case i.Var1 == "":
+ return fmt.Errorf("if was declared but VAR1 was not supplied in createIf. Raw %s", *a)
+
+ case i.Var2 == "" && i.Type != "exists":
+ return fmt.Errorf("if was declared but VAR2 was not supplied in createIf. Raw %s", *a)
+ }
+
+ for _, c := range allowedConditions {
+ if i.Type == c {
+ s.conditions = append(s.conditions, *i)
+ return nil
+ }
+ }
+
+ return fmt.Errorf("if was declared but the supplied TYPE is not supported. Supported types are %s in createId. Raw %s", allowedConditions, *a)
+}
diff --git a/jobs_run.go b/jobs_run.go
new file mode 100644
index 0000000..aa7ca58
--- /dev/null
+++ b/jobs_run.go
@@ -0,0 +1,208 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "time"
+)
+
+// workerFetch is called upon when the Server is started with *Server.Start.
+// The number of worker spawned is determined by the value supplied in the *Server.New call.
+// It fetches by listening to the *Server.parsedJobs channel and calling the *Server.fetchJob
+// function for each job sent. The result will be added to a result slice and sent to the
+// *Server.resultJobs channel.
+func (srv *Server) workerFetch() {
+ srv.wgRes.Add(1)
+ results := []*Result{}
+
+ for srv.running {
+ j := <-srv.parsedJobs
+ if j == nil {
+ break
+ }
+
+ res := srv.fetchJob(j)
+ results = append(results, res)
+ srv.resultCounterChan <- 1
+ srv.wgRun.Done()
+ }
+
+ srv.resultJobs <- results
+}
+
+// fetchJob will loop through the job j steps and call *job.fetchStep through the *job.runFetchJob on each iteration.
+// Any errors will be added to the results error value.
+// Returns *result.
+func (srv *Server) fetchJob(j *job) *Result {
+ r := &Result{StartTime: time.Now()}
+
+ for i := 0; i < len(j.steps); i++ {
+ switch {
+ // If a for loop is detected, we must run multiple steps inside a single step.
+ // First we must replace any variables in the in data for the for loop (raw values).
+ // Since these can be based on results from a body, header etc.
+ // After we have run the replaceFromVariablesForLoop and the value slice is set we
+ // can iterate over the slice and set the for variable to the current value and
+ // run all steps in the for loop with that value.
+ // v = index of for variable, s = index for step in the for loop.
+ case j.steps[i].forloop.varname != "":
+ j.replaceFromVariablesForLoop(&j.steps[i])
+
+ for v := range j.steps[i].forloop.values {
+ // Set the variable to be used for this iteration of the for loop.
+ j.vars[j.steps[i].forloop.varname] = j.steps[i].forloop.values[v]
+
+ for s := range j.steps[i].forloop.steps {
+ res, err := j.runFetchJob(srv.fetchFunc, j.steps[i].forloop.steps[s].deepCopyStep())
+ r.Steps = append(r.Steps, res)
+ r.Status = res.Status
+
+ if err != nil {
+ r.Err = err
+ break
+ }
+ }
+ }
+
+ // The default fetching method, when we just have normal global steps (ie, not in a for loop).
+ default:
+ res, err := j.runFetchJob(srv.fetchFunc, &j.steps[i])
+ r.Steps = append(r.Steps, res)
+ r.Status = res.Status
+
+ if err != nil {
+ r.Err = err
+ break
+ }
+ }
+
+ // If any errors where set above, we should not do any more steps.
+ // And just break out of the for loop and save the results.
+ if r.Err != nil {
+ break
+ }
+ }
+
+ r.Duration = time.Now().Sub(r.StartTime)
+
+ return r
+}
+
+// deepCopyStep is used to make a deep copy of a step. Which means that we will copy every array/map it contains
+// so that every step can be run independently of another. Otherwise changes to one step on data structures that
+// are referenced by memory, such as slices, maps will be updated when we replace vars and such. Which is not
+// what we want when running multiple steps inside a step (for loops).
+// So we make a deep copy of all the data from step s and return a new step.
+// Returns *step.
+func (s *step) deepCopyStep() *step {
+ newStep := &step{
+ method: s.method,
+ forloop: forloop{}, // No nested for loop support, so should be empty.
+ auth: s.auth,
+ url: s.url,
+ body: s.body,
+ }
+
+ // Make copy of conditions/if slice.
+ for _, i := range s.conditions {
+ newStep.conditions = append(newStep.conditions, i)
+ }
+
+ // Make copy of varfrom slice.
+ for _, v := range s.varfrom {
+ newStep.varfrom = append(newStep.varfrom, v)
+ }
+
+ // Make copy of headers slice.
+ for _, h := range s.headers {
+ newStep.headers = append(newStep.headers, h)
+ }
+
+ // Make copy of cookies.
+ for _, c := range s.cookies {
+ newStep.cookies = append(newStep.cookies, c)
+ }
+
+ return newStep
+}
+
+// runFetchJob will run the actual fetchStep function on the step s with
+// func(*http.Request) (*http.Response, error) c. This is split out so that
+// the function can be used both for iterations over a for loop
+// (multiple steps within a step) or just a basic single step.
+// Returns *ResultSteps and *ResultError.
+func (j *job) runFetchJob(c func(*http.Request) (*http.Response, error), s *step) (*ResultStep, *ResultError) {
+ // Dont run fetch on steps with no URL.
+ if s.url == "" {
+ return &ResultStep{}, nil
+ }
+
+ stepStart := time.Now()
+ status, err := j.fetchStep(c, s)
+
+ res := &ResultStep{
+ Method: s.method,
+ URL: s.url,
+ Headers: s.headers,
+ Cookies: s.cookies,
+ Body: s.body,
+ StartTime: stepStart,
+ Duration: time.Now().Sub(stepStart),
+ Status: status,
+ }
+
+ if err != nil {
+ err.Step = res
+ return res, err
+ }
+
+ return res, nil
+}
+
+// fetchStep will make an request against the steps url method.
+// If any of the if/conditions are matched we will not fetch anything and directly return a statusCode of 0.
+// We will replace any variables from the URL, Body Header and Cookies with the *job.replaceFromVariables.
+// Auth, Headers and Cookies are then added to the request addMetaData function.
+// Will return the statusCode of the request as well as any error. The error will include the
+// step which failed including all the data so it can be easily tracked in logfiles.
+// Any response status code 400 or above will result in an error.
+// Returns int and *ResultError.
+func (j *job) fetchStep(c func(*http.Request) (*http.Response, error), s *step) (int, *ResultError) {
+ if !j.checkConditions(s) {
+ return 0, nil
+ }
+
+ j.replaceFromVariables(s)
+
+ req, err := http.NewRequest(s.method, s.url, bytes.NewBuffer([]byte(s.body)))
+ if err != nil {
+ return -1, &ResultError{Error: fmt.Errorf("Error creating up the Request in *job.fetchStep. %s", err)}
+ }
+ j.addOptions(s, req)
+
+ res, err := c(req)
+ if err != nil {
+ return -1, &ResultError{Error: fmt.Errorf("Error sending the Request in *job.fetchStep. %s", err)}
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode > 399 {
+ body, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ body = []byte("")
+ }
+ return res.StatusCode, &ResultError{Error: fmt.Errorf("%d %s %s", res.StatusCode, s.method, s.url), URL: s.url, Status: res.StatusCode, Body: string(body)}
+ }
+
+ j.appendResponseCookiesToJob(res)
+
+ err = j.variablesFrom(s, res)
+ if err != nil {
+ return -1, &ResultError{Error: err}
+ }
+
+ return res.StatusCode, nil
+}
diff --git a/replace_from_vars.go b/replace_from_vars.go
new file mode 100644
index 0000000..b767601
--- /dev/null
+++ b/replace_from_vars.go
@@ -0,0 +1,102 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+)
+
+const (
+ replaceVarSyntax = "{{%s}}" // Syntax to search for when replacing variables.
+ searchSyntax = "{{StepTestSyntax}}" // searchSyntax used by VARFROM to look for patterns in BODY/HEADER.
+ searchSyntaxReplace = ").+(" // searchSyntaxReplace is what we replace searchSyntax with in our regular expression.
+ searchSyntaxRegexp = "(%s)" // searchSyntaxRegexp is what we encapsulate the whole search string to make a regular expression.
+)
+
+// replaceFromVariables will run replacement functions on data based on the variables stored in job j.
+// It will replace the variables found in either URL, Body, Headers or Cookies with those stored in the jobs variables.
+func (j *job) replaceFromVariables(s *step) {
+ // Replace from variables.
+ s.headers = append(j.globalHeaders, s.headers...)
+ s.cookies = append([]http.Cookie{}, j.cookies...)
+
+ for n, v := range j.vars {
+ j.varReplaceURL(s, &n, &v)
+ j.varReplaceBody(s, &n, &v)
+ j.varReplaceHeaders(s, &n, &v)
+ j.varReplaceCookies(s, &n, &v)
+ }
+}
+
+// varReplaceURL will replace every occurrence of name n with value v in the URL.
+func (*job) varReplaceURL(s *step, n *string, v *string) {
+ s.url = strings.Replace(s.url, fmt.Sprintf(replaceVarSyntax, *n), *v, -1)
+}
+
+// varReplaceBody will replace every occurrence of name n with value v in the Body.
+func (*job) varReplaceBody(s *step, n *string, v *string) {
+ s.body = strings.Replace(s.body, fmt.Sprintf(replaceVarSyntax, *n), *v, -1)
+}
+
+// varReplaceHeaders will append the jobs j global Headers with the steps s local Headers and
+// replace every occurrence of name n with value v in the headers.
+// When done it will overwrite the the steps header value with the combined and replaced headers.
+func (j *job) varReplaceHeaders(s *step, n *string, v *string) {
+ for i := range s.headers {
+ s.headers[i].Value = strings.Replace(s.headers[i].Value, fmt.Sprintf(replaceVarSyntax, *n), *v, -1)
+ }
+}
+
+// varReplaceCookies will copy the jobs cookies and replace every occurrence of name n with value v in cookies.
+// When done it will overwrite the the steps cookies value with the replaced cookies.
+func (j *job) varReplaceCookies(s *step, n *string, v *string) {
+ for i := range s.cookies {
+ s.cookies[i].Value = strings.Replace(s.cookies[i].Value, fmt.Sprintf(replaceVarSyntax, *n), *v, -1)
+ }
+}
+
+// replaceFromVariablesForLoop will run replacement functions on FOR variables on the arrays and variables stored in job j.
+// It will first try to match any array with the name specified and replace the for loops values with that array.
+// After that it will run variable replacement on the array. So it's possible to store variables in the array.
+// It will replace the variables found in either URL, Body, Headers or Cookies with those stored in the jobs variables.
+func (j *job) replaceFromVariablesForLoop(s *step) {
+ for n, v := range j.arrays {
+ j.replaceForLoopArray(s, &n, v)
+ }
+
+ for n, v := range j.vars {
+ j.replaceForLoopStrings(s, &n, &v)
+ }
+}
+
+// replaceForLoopStrings will replace every occurrence of name n with value v in the For Loops values.
+// If v exists and can be unmarshal from a JSON array we will append them to values.
+// It it's a regular string we will replace it.
+func (*job) replaceForLoopStrings(s *step, n *string, v *string) {
+ for i, storedValue := range s.forloop.values {
+ if strings.Contains(storedValue, fmt.Sprintf(replaceVarSyntax, *n)) {
+ arr := new([]string)
+ err := json.Unmarshal([]byte(*v), arr)
+
+ if err != nil {
+ s.forloop.values[i] = *v
+ continue
+ }
+
+ // Stuff the new slice in the same place that the old variable was.
+ *arr = append(s.forloop.values[:i], *arr...)
+ s.forloop.values = append(*arr, s.forloop.values[i+1:]...)
+ }
+ }
+}
+
+// replaceForLoopArrays will replace a array name with the associated array.
+func (*job) replaceForLoopArray(s *step, n *string, v []string) {
+ for _, storedValue := range s.forloop.values {
+ if strings.Contains(storedValue, fmt.Sprintf(replaceVarSyntax, *n)) {
+ s.forloop.values = v
+ }
+ }
+}
diff --git a/results.go b/results.go
new file mode 100644
index 0000000..a9216f4
--- /dev/null
+++ b/results.go
@@ -0,0 +1,22 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+// workerResults will listen for []*results to be sent on the *Server.resultJobs channel.
+// It will append any data from the channel to *Server.results.
+func (srv *Server) workerResults() {
+ for {
+ results := <-srv.resultJobs
+
+ srv.results = append(srv.results, results...)
+ srv.wgRes.Done()
+ }
+}
+
+// workerResultsCounter will listen for incoming increment changes on the *Server.resultCounterChan channel
+// and add those to the *Server.resultsCounter. This way we can track the amount of finished requests.
+func (srv *Server) workerResultsCounter() {
+ for {
+ inc := <-srv.resultCounterChan
+ srv.resultsCounter += inc
+ }
+}
diff --git a/server.go b/server.go
new file mode 100644
index 0000000..e31ec9d
--- /dev/null
+++ b/server.go
@@ -0,0 +1,185 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "fmt"
+ "net/http"
+ "time"
+)
+
+// New takes a number of virtual user v and request timeout t and creates a StepTest Server.
+// If c is not nil that function will be used for all requests. This is usefull when you
+// need to sign your requests or do anything else fancy with the them :) :) :).
+// If c is set timeout will be ignored for obvious reasons, so please handle this in your
+// own function if that is the case.
+// Returns *Server and error.
+func New(v int, t int, c func(*http.Request) (*http.Response, error)) (*Server, error) {
+ // Default to 100 workers.
+ if v < 1 {
+ v = 100
+ }
+
+ // Default to 30000ms in fetch timeout.
+ if t < 1 {
+ t = 30000
+ }
+
+ // Default to a simple function if no one was specified.
+ if c == nil {
+ client := &http.Client{Timeout: time.Duration(t) * time.Millisecond}
+ c = func(req *http.Request) (*http.Response, error) { return client.Do(req) }
+ }
+
+ srv := &Server{
+ fetchWorkers: v,
+ fetchFunc: c,
+ addedJobsCounterChan: make(chan int),
+ parsedJobs: make(chan *job),
+ resultJobs: make(chan []*Result),
+ resultCounterChan: make(chan int),
+ }
+
+ return srv, nil
+}
+
+// Start will start the execution of parsed jobs. If there are any un-parsed jobs left
+// in the queue it will wait for them to finish before starting execution.
+func (srv *Server) Start() {
+ go srv.workerResults()
+ go srv.workerResultsCounter()
+ go srv.workerAddedJobCounter()
+
+ srv.running = true
+ srv.startTime = time.Now()
+
+ for i := 0; i < srv.fetchWorkers; i++ {
+ go srv.workerFetch()
+ }
+}
+
+// StopRunning will send a signal to stop fetching requests on the Server.
+// StopRunning can only be called when the Server is in a IsRunning -state.
+// Returns error.
+//FIXME: We need to re-implement this so we can stop a running program without getting send on closed channel.
+func (srv *Server) StopRunning() error {
+ switch {
+ case !srv.IsRunning():
+ return fmt.Errorf("Failed to StopRunning. The Server isn't in a Running state")
+ }
+
+ srv.closeRunning()
+ return nil
+}
+
+// WaitDone will wait until the Server has finished fetching all the requests in the *Server.jobs map.
+// WaitDone will block the program until it has finished.
+func (srv *Server) WaitDone() {
+ srv.wgRun.Wait()
+ srv.closeRunning()
+}
+
+// closeRunning will close everything associated with the Server in running state.
+func (srv *Server) closeRunning() {
+ srv.stopping = true
+ srv.running = false
+ close(srv.parsedJobs)
+ srv.wgRes.Wait()
+ srv.endTime = time.Now()
+}
+
+// GetNumberOfVirtualUsers returns the number of virtual users.
+// Returns int.
+func (srv *Server) GetNumberOfVirtualUsers() int {
+ return srv.fetchWorkers
+}
+
+// GetNumberOfJobs returns the number of jobs stored on the Server.
+// Returns int
+func (srv *Server) GetNumberOfJobs() int {
+ return srv.addedJobsCounter
+}
+
+// GetResults will return the all the results.
+// Returns []*Result.
+func (srv *Server) GetResults() []*Result {
+ return srv.results
+}
+
+// GetSteps will return all the steps from a *Result.
+// Returns []*ResultStep.
+func (res *Result) GetSteps() []*ResultStep {
+ return res.Steps
+}
+
+// GetNumberOfRequests returns the number of successfull requests.
+// Returns int.
+func (srv *Server) GetNumberOfRequests() int {
+ return srv.resultsCounter
+}
+
+// GetError will return any error from a *Result.
+// Returns []*ResultError.
+func (res *Result) GetError() *ResultError {
+ return res.Err
+}
+
+// GetNumberOfErrors will return the amount of requests that had an error.
+// Returns int.
+func (srv *Server) GetNumberOfErrors() int {
+ errors := 0
+ for _, res := range srv.results {
+ if res.Err != nil {
+ errors++
+ }
+ }
+ return errors
+}
+
+// GetErrorMessages will return all the errors since the Server was started.
+// Returns []*ResultError.
+func (srv *Server) GetErrorMessages() []*ResultError {
+ errors := []*ResultError{}
+ for _, res := range srv.results {
+ if res.Err != nil {
+ errors = append(errors, res.Err)
+ }
+ }
+ return errors
+}
+
+// GetAverageFetchTime will return the average fetch time for all the requests. Requests that resulted in errors will be ignored in the average.
+// Returns time.Duration.
+func (srv *Server) GetAverageFetchTime() time.Duration {
+ duration := int64(0)
+ for _, res := range srv.results {
+ if res.Err != nil {
+ continue
+ }
+
+ duration += int64(res.Duration)
+ }
+
+ numRequests := int64(len(srv.results))
+ if duration == 0 || numRequests == 0 {
+ return 0
+ }
+
+ return time.Duration(duration/numRequests) / time.Millisecond
+}
+
+// IsRunning returns true if the Server is still running jobs. False if it has finished or manually been stopped.
+// Returns bool.
+func (srv *Server) IsRunning() bool {
+ return srv.running
+}
+
+// GetTotalRunTime will return the total runtime since Server start.
+// Returns time.Duration.
+func (srv *Server) GetTotalRunTime() time.Duration {
+ duration := srv.endTime.Sub(srv.startTime)
+ if duration == 0 {
+ return 0
+ }
+
+ return duration / time.Second
+}
diff --git a/server_test.go b/server_test.go
new file mode 100644
index 0000000..734e9cc
--- /dev/null
+++ b/server_test.go
@@ -0,0 +1,150 @@
+package steptest
+
+import (
+ "testing"
+ "time"
+)
+
+func TestNew(t *testing.T) {
+ srv, err := New(50, 15000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ // Check the number of fetchWorkers
+ if srv.fetchWorkers != 50 {
+ t.Errorf("Wrong number of workers. Expected %d but got %d", 50, srv.fetchWorkers)
+ }
+
+ if srv.addedJobsCounterChan == nil {
+ t.Error("*Server.addedJobsCounterChan was nil. Expected chan int but got nil")
+ }
+
+ if srv.parsedJobs == nil {
+ t.Error("*Server.parsedJobs was nil. Expected chan *job but got nil")
+ }
+
+ if srv.resultJobs == nil {
+ t.Error("*Server.resultJobs was nil. Expected chan []*Result but got nil")
+ }
+
+ if srv.resultCounterChan == nil {
+ t.Error("*Server.resultCounterChan was nil. Expected chan in but got nil")
+ }
+
+ if srv.fetchFunc == nil {
+ t.Error("*Server.fetchFunc was nil. Expected func(*http.Request) (*http.Response, error) but got nil")
+ }
+}
+
+func TestStart(t *testing.T) {
+ srv, err := New(1, 30000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ srv.Start()
+ if srv.running != true {
+ t.Error("Server isn't running. Expected *Server.running to be true")
+ }
+}
+
+func TestStopRunning(t *testing.T) {
+ srv, err := New(1, 30000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ srv.Start()
+ err = srv.StopRunning()
+ if err != nil {
+ t.Error(err)
+ }
+
+ if srv.running != false {
+ t.Errorf("Server is still in running state. Expected *Server.running to be false")
+ }
+
+ if srv.stopping != true {
+ t.Errorf("Server is not in stopping state. Expected *Server.stopping to be true")
+ }
+}
+
+func TestWaitDone(t *testing.T) {
+ srv, err := New(1, 30000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ srv.Start()
+ srv.WaitDone()
+
+ if srv.running != false {
+ t.Errorf("Server is still in running state. Expected *Server.running to be false")
+ }
+
+ if srv.stopping != true {
+ t.Errorf("Server is not in stopping state. Expected *Server.stopping to be true")
+ }
+}
+
+func TestGetNumberOfVirtualUsers(t *testing.T) {
+ srv, err := New(153, 30000, nil)
+ if err != nil {
+ t.Error(err)
+ }
+
+ if srv.GetNumberOfVirtualUsers() != 153 {
+ t.Errorf("Wrong number of Virtual Users. Expected %d but got %d", 153, srv.GetNumberOfVirtualUsers())
+ }
+}
+
+func TestGetNumberOfJobs(t *testing.T) {
+ srv := &Server{addedJobsCounter: 12}
+
+ if srv.GetNumberOfJobs() != 12 {
+ t.Errorf("Wrong number of Added Jobs. Expected %d but got %d", 12, srv.GetNumberOfJobs())
+ }
+}
+
+func TestGetResults(t *testing.T) {
+ start := time.Now()
+ end := start.Sub(start.Add(time.Duration(10) * time.Second))
+ srv := &Server{
+ results: []*Result{
+ &Result{
+ StartTime: start,
+ Duration: end,
+ Err: nil,
+ Status: 200,
+ Steps: nil,
+ },
+ },
+ }
+
+ res := srv.GetResults()
+
+ if len(res) != 1 {
+ t.Errorf("Wrong number of Results. Expected %d but got %d", 1, len(res))
+ }
+
+ if res[0].StartTime != start {
+ t.Errorf("Wrong StartTime. Expected %s but got %s", start.String(), res[0].StartTime.String())
+ }
+
+ if res[0].Duration != end {
+ t.Errorf("Wrong StartTime. Expected %s but got %s", end.String(), res[0].Duration.String())
+ }
+
+ if res[0].Err != nil {
+ t.Errorf("Err was not nil")
+ }
+
+ if res[0].Status != 200 {
+ t.Errorf("Wrong Status. Expected %d but got %d", 200, res[0].Status)
+ }
+
+ if res[0].Steps != nil {
+ t.Errorf("Steps was not nil")
+ }
+}
diff --git a/set_var_from.go b/set_var_from.go
new file mode 100644
index 0000000..9b750c9
--- /dev/null
+++ b/set_var_from.go
@@ -0,0 +1,74 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "regexp"
+ "strings"
+)
+
+// variablesFrom will set variables from either BODY or HEADERS as defined in step s
+// from the response res and add them to the job j.
+// Returns error.
+func (j *job) variablesFrom(s *step, res *http.Response) error {
+ if len(s.varfrom) == 0 {
+ return nil
+ }
+
+ raw, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return fmt.Errorf("Couldn't read Body of response in *job.variablesFrom. %s", err.Error())
+ }
+ headers := res.Header
+
+ for _, v := range s.varfrom {
+ switch strings.ToUpper(v.From) {
+ case "BODY":
+ err := j.variableFromBody(&v, &raw)
+ if err != nil {
+ return err
+ }
+
+ case "HEADER":
+ j.variableFromHeader(&v, headers)
+ }
+ }
+
+ return nil
+}
+
+// variableFromHeader will create or overwrite a variable in the jobs j vars map based on the
+// value stored in the response res headers of the header with name from v.orgSyntax.
+func (j *job) variableFromHeader(v *varfromItem, header http.Header) {
+ value := header.Get(v.OrgSyntax)
+ if value == "" {
+ return
+ }
+
+ j.vars[v.Varname] = value
+}
+
+// variableFromBody will create or overwrite a variable in the jobs j vars map based on the
+// search syntax supplied by v.syntax.
+// Returns error.
+func (j *job) variableFromBody(v *varfromItem, raw *[]byte) error {
+ regexp, err := regexp.Compile(v.Syntax)
+ if err != nil {
+ return fmt.Errorf("Couldn't compile regular expression in *job.variableFromBody. %s", err.Error())
+ }
+
+ value := string(regexp.Find(*raw))
+ if value == "" {
+ return nil
+ }
+
+ parts := strings.Split(v.OrgSyntax, searchSyntax)
+ for _, part := range parts {
+ value = strings.Replace(value, part, "", -1)
+ }
+
+ j.vars[v.Varname] = value
+ return nil
+}
diff --git a/structs.go b/structs.go
new file mode 100644
index 0000000..aa06ca9
--- /dev/null
+++ b/structs.go
@@ -0,0 +1,150 @@
+// Package steptest makes transactional load test easy.
+package steptest
+
+import (
+ "net/http"
+ "sync"
+ "time"
+)
+
+// Server contains the necessary functions and data to run StepTest.
+// Should be instantiated with New function.
+type Server struct {
+ fetchWorkers int
+ fetchFunc func(*http.Request) (*http.Response, error)
+
+ startTime time.Time
+ endTime time.Time
+
+ addedJobsCounter int
+ addedJobsCounterChan chan int
+
+ parsedJobs chan *job
+
+ results []*Result
+ resultJobs chan []*Result
+ resultsCounter int
+ resultCounterChan chan int
+
+ stopping bool
+ running bool
+ wgRun sync.WaitGroup
+ wgRes sync.WaitGroup
+}
+
+type rawJob struct {
+ steps string
+ vars map[string]string
+}
+
+type job struct {
+ steps []step
+ vars map[string]string
+ arrays map[string][]string
+ globalHeaders []header
+ globalAuth auth
+ cookies []http.Cookie
+
+ // For variables. The addTo contains which step index to add sub steps to. For now we only use one value in the slice
+ // since nested for loops are not supported.
+ forcounter int
+ forRemoveNextStep bool
+ addTo []int
+}
+
+type step struct {
+ method string
+ headers []header
+
+ forloop forloop
+
+ conditions []condition
+
+ // Only used for storing results of replaced cookies. All cookies are global.
+ cookies []http.Cookie
+ auth auth
+ url string
+ body string
+ varfrom []varfromItem
+}
+
+type forloop struct {
+ varname string
+ values []string
+
+ steps []step
+}
+
+type cookie struct {
+ Name string `json:"name"`
+ Value string `json:"value"`
+ Path string `json:"path"`
+ Domain string `json:"domain"`
+ Expires time.Time `json:"expires"`
+ MaxAge int `json:"maxAge"`
+ Secure bool `json:"secure"`
+ HTTPOnly bool `json:"httpOnly"`
+}
+
+type variable struct {
+ Name string `json:"name"`
+ Value string `json:"value"`
+}
+
+type array struct {
+ Name string `json:"name"`
+ Values []string `json:"values"`
+}
+
+type auth struct {
+ Username string `json:"username"`
+ Password string `json:"password"`
+}
+
+type header struct {
+ Name string `json:"Name"`
+ Value string `json:"Value"`
+}
+
+type varfromItem struct {
+ From string `json:"from"`
+ Varname string `json:"name"`
+ OrgSyntax string `json:"find"`
+ Syntax string `json:"-"`
+}
+
+type condition struct {
+ Type string `json:"type"`
+ Var1 string `json:"var1"`
+ Var2 string `json:"var2"`
+}
+
+// Result contains the result of a job.
+type Result struct {
+ StartTime time.Time `json:"startTime"`
+ Status int `json:"status"`
+ Duration time.Duration `json:"duration"`
+ Steps []*ResultStep `json:"steps"`
+ Err *ResultError `json:"error"`
+}
+
+// ResultStep contains the processed step results.
+type ResultStep struct {
+ StartTime time.Time `json:"startTime"`
+ Status int `json:"status"`
+ Duration time.Duration `json:"duration"`
+ Method string `json:"method"`
+ URL string `json:"url"`
+ Headers []header `json:"headers"`
+ Cookies []http.Cookie `json:"cookies"`
+ Body string `json:"body"`
+}
+
+// ResultError contains the error and the step of the error.
+type ResultError struct {
+ Error error `json:"error"`
+ URL string `json:"url"`
+ Status int `json:"status"`
+ Body string `json:"body"`
+ Step *ResultStep `json:"step"`
+}
diff --git a/todo b/todo
new file mode 100644
index 0000000..4717e14
--- /dev/null
+++ b/todo
@@ -0,0 +1,15 @@
+Add test for
+
+
+finished files
+ add_options.go
+ check_conditions.go
+ jobs_add.go
+ jobs_add_test.go
+ jobs_parse.go
+ jobs_run.go
+ replace_from_vars.go
+ results.go
+ ----
+ set_var_from.go
+ structs.go
\ No newline at end of file