From f8599c124384eec6227dd75a10ecf4bf9c0872f6 Mon Sep 17 00:00:00 2001 From: Geremia Taglialatela Date: Thu, 12 Sep 2024 13:47:48 +0200 Subject: [PATCH] Fix whitespace related offenses --- bin/autoheathen | 24 +++--- bin/cvheathen | 24 +++--- bin/docpath | 8 +- config.ru | 2 +- lib/app.rb | 75 ++++++++++--------- lib/autoheathen/config.rb | 4 +- lib/autoheathen/email_processor.rb | 29 ++++--- lib/config.rb | 2 +- lib/converter.rb | 10 +-- lib/doc_key.rb | 2 +- lib/document.rb | 52 +++++++------ lib/errors.rb | 14 +++- lib/heathen.rb | 2 +- lib/heathen/converter.rb | 4 +- lib/heathen/errors.rb | 12 +-- lib/heathen/executioner.rb | 23 +++--- lib/heathen/filename.rb | 2 +- lib/heathen/job.rb | 6 +- lib/heathen/processor.rb | 10 +-- .../processor_methods/convert_image.rb | 5 +- lib/heathen/processor_methods/libreoffice.rb | 8 +- lib/heathen/processor_methods/tesseract.rb | 1 + lib/heathen/processor_methods/wkhtmltopdf.rb | 45 +++++------ lib/heathen/task.rb | 38 +++++----- lib/legacy_converter.rb | 3 +- lib/sidekiq_workers.rb | 14 ++-- lib/utils.rb | 20 ++--- spec/heathen/convert_spec.rb | 3 +- spec/heathen/filename_spec.rb | 1 - spec/lib/app_spec.rb | 28 +++---- spec/lib/autoheathen/config_spec.rb | 28 +++---- spec/lib/autoheathen/email_processor_spec.rb | 22 +++--- spec/lib/converter_spec.rb | 2 +- spec/lib/doc_key_spec.rb | 2 +- spec/lib/document_spec.rb | 28 +++---- spec/lib/sidekiq_workers_spec.rb | 8 +- spec/lib/utils_spec.rb | 11 ++- spec/spec_helper.rb | 2 +- 38 files changed, 299 insertions(+), 275 deletions(-) diff --git a/bin/autoheathen b/bin/autoheathen index 06f7628..076dc07 100755 --- a/bin/autoheathen +++ b/bin/autoheathen @@ -29,21 +29,21 @@ mode = :return_to_sender mail_to = nil OptionParser.new do |opts| - opts.on( '-r', '--return-to-sender', 'Converted files will be emailed back to sender' ) { mode = :return_to_sender } - opts.on( '-t', '--to EMAIL', 'Converted files will be emailed to this address' ) { |e| mode = :email; mail_to = e } - opts.on( '-s', '--summary', 'Don\'t deposit the converted file, just log a summary' ) { cfg[:deliver] = false } - opts.on( '-l', '--language', 'Document language' ) { |l| cfg[:language] = l } - opts.on( '-M', '--mail-host MAILHOST', 'Mail server for sending replies' ) { |h| cfg[:mail_host] = h } - opts.on( '-P', '--mail-port PORT', Integer, 'Mail server port' ) { |p| cfg[:mail_port] = p } - opts.on( '-C', '--config FILE', 'Configuration YAML file' ) { |file| cfg[:config_file] = file } - opts.on( '-v', '--verbose', 'Running commentary' ) { cfg[:logger] = Logger.new(STDOUT) } + opts.on('-r', '--return-to-sender', 'Converted files will be emailed back to sender') { mode = :return_to_sender } + opts.on('-t', '--to EMAIL', 'Converted files will be emailed to this address') { |e| mode = :email; mail_to = e } + opts.on('-s', '--summary', 'Don\'t deposit the converted file, just log a summary') { cfg[:deliver] = false } + opts.on('-l', '--language', 'Document language') { |l| cfg[:language] = l } + opts.on('-M', '--mail-host MAILHOST', 'Mail server for sending replies') { |h| cfg[:mail_host] = h } + opts.on('-P', '--mail-port PORT', Integer, 'Mail server port') { |p| cfg[:mail_port] = p } + opts.on('-C', '--config FILE', 'Configuration YAML file') { |file| cfg[:config_file] = file } + opts.on('-v', '--verbose', 'Running commentary') { cfg[:logger] = Logger.new(STDOUT) } end.parse! email = Mail.read_from_string $stdin.read processor = AutoHeathen::EmailProcessor.new(cfg) case mode - when :return_to_sender - processor.process_rts email - when :email - processor.process email, mail_to +when :return_to_sender + processor.process_rts email +when :email + processor.process email, mail_to end diff --git a/bin/cvheathen b/bin/cvheathen index f34fd8b..498fead 100755 --- a/bin/cvheathen +++ b/bin/cvheathen @@ -22,16 +22,16 @@ verbose = false mode = :file OptionParser.new do |opts| - opts.on( '-a', '--action ACTION', 'Action to perform' ) { |a| action = a } - opts.on( '-f', '--file FILENAME', 'Input file or directory' ) { |f| in_file = Pathname.new(f) } - opts.on( '-o', '--outfile FILENAME', 'Output file or directory' ) { |f| out_file = Pathname.new(f) } - opts.on( '-r', '--recurse', 'Recurse through input directory' ) { recurse = true } - opts.on( '-l', '--language LANG', 'Language of the input file(s)' ) { |l| language = l } - opts.on( '-v', '--verbose', 'Verbose output' ) { verbose = true } - opts.on( '-h', '--help', 'This message' ) { puts opts; exit } + opts.on('-a', '--action ACTION', 'Action to perform') { |a| action = a } + opts.on('-f', '--file FILENAME', 'Input file or directory') { |f| in_file = Pathname.new(f) } + opts.on('-o', '--outfile FILENAME', 'Output file or directory') { |f| out_file = Pathname.new(f) } + opts.on('-r', '--recurse', 'Recurse through input directory') { recurse = true } + opts.on('-l', '--language LANG', 'Language of the input file(s)') { |l| language = l } + opts.on('-v', '--verbose', 'Verbose output') { verbose = true } + opts.on('-h', '--help', 'This message') { puts opts; exit } end.parse! -logger = Logger.new( verbose ? STDOUT : nil ) +logger = Logger.new(verbose ? STDOUT : nil) converter = Heathen::Converter.new logger: logger files = [] @@ -39,16 +39,17 @@ files = [] if in_file.file? abort "Output is a directory, but expected a file" if out_file.directory? content = converter.convert action, File.read(in_file), language - File.open( out_file, 'wb' ) { |f| f.write content } + File.open(out_file, 'wb') { |f| f.write content } elsif in_file.directory? abort "Invalid output directory" unless out_file.directory? - Dir.glob( in_file + ( recurse ? '**/*' : '*') ).each do |file| + Dir.glob(in_file + (recurse ? '**/*' : '*')).each do |file| next unless Pathname.new(file).file? + begin content = converter.convert action, File.read(file), language new_file = Heathen::Filename.suggest_in_new_dir file, content.mime_type, in_file.to_s, out_file.to_s Pathname.new(new_file).parent.mkpath - File.open( new_file, 'wb' ) { |f| f.write content } + File.open(new_file, 'wb') { |f| f.write content } rescue StandardError => e logger.error "Failed to convert #{file}: #{e.message}" end @@ -56,4 +57,3 @@ elsif in_file.directory? else abort "Invalid input file" unless in_file.file? end - diff --git a/bin/docpath b/bin/docpath index f62df2a..1627373 100755 --- a/bin/docpath +++ b/bin/docpath @@ -17,10 +17,10 @@ storage_dir = nil config_file = BASE + 'config' + 'app.yml' OptionParser.new do |opts| - opts.on( '-a', '--app APP', 'Name of the app' ) { |a| app = a } - opts.on( '-d', '--doc-id DOCID', 'Document ID' ) { |d| doc_id = d } - opts.on( '-s', '--storage-dir DIR', 'Storage directory' ) { |d| storage_dir = d } - opts.on( '-h', '--help', 'This message' ) { puts opts; exit } + opts.on('-a', '--app APP', 'Name of the app') { |a| app = a } + opts.on('-d', '--doc-id DOCID', 'Document ID') { |d| doc_id = d } + opts.on('-s', '--storage-dir DIR', 'Storage directory') { |d| storage_dir = d } + opts.on('-h', '--help', 'This message') { puts opts; exit } opts.separator '' opts.separator 'Examples:' opts.separator ' docpath -a myapp -d 123456 -s /opt/storage' diff --git a/config.ru b/config.ru index c6709ae..a255904 100644 --- a/config.ru +++ b/config.ru @@ -5,7 +5,7 @@ require 'pathname' require "sinatra" -BASE=Pathname.new(__FILE__).realpath.parent +BASE = Pathname.new(__FILE__).realpath.parent $: << BASE $: << BASE + 'lib' require 'config/initializers/sidekiq' diff --git a/lib/app.rb b/lib/app.rb index 6d537a4..7336bc8 100644 --- a/lib/app.rb +++ b/lib/app.rb @@ -13,7 +13,7 @@ module Colore class App < Sinatra::Base set :backtrace, true before do - @storage_dir = Pathname.new( C_.storage_directory ) + @storage_dir = Pathname.new(C_.storage_directory) @legacy_url_base = C_.legacy_url_base || url('/') @logger = Logger.new(C_.conversion_log || STDOUT) @errlog = Logger.new(C_.error_log || STDERR) @@ -42,12 +42,12 @@ class App < Sinatra::Base # - callback_url # - file # - author - put '/document/:app/:doc_id/:filename' do |app,doc_id,filename| + put '/document/:app/:doc_id/:filename' do |app, doc_id, filename| begin - doc_key = DocKey.new app,doc_id + doc_key = DocKey.new app, doc_id doc = Document.create @storage_dir, doc_key # will raise if doc exists doc.title = params[:title] if params[:title] - call env.merge( 'REQUEST_METHOD'=>'POST' ) + call env.merge('REQUEST_METHOD' => 'POST') rescue StandardError => e respond_with_error e end @@ -64,11 +64,12 @@ class App < Sinatra::Base # - callback_url # - file # - author - post '/document/:app/:doc_id/:filename' do |app,doc_id,filename| + post '/document/:app/:doc_id/:filename' do |app, doc_id, filename| begin - doc_key = DocKey.new app,doc_id - doc = Document.load( @storage_dir, doc_key ) + doc_key = DocKey.new app, doc_id + doc = Document.load(@storage_dir, doc_key) raise InvalidParameter.new :file unless params[:file] + version = doc.new_version do |version| doc.add_file version, filename, params[:file][:tempfile], params[:author] doc.set_current version @@ -86,7 +87,7 @@ class App < Sinatra::Base respond 201, "Document stored", { app: app, doc_id: doc_id, - path: doc.file_path( Colore::Document::CURRENT, filename ), + path: doc.file_path(Colore::Document::CURRENT, filename), } rescue StandardError => e respond_with_error e @@ -94,10 +95,10 @@ class App < Sinatra::Base end # Updates the document title - post '/document/:app/:doc_id/title/:title' do |app,doc_id,title| + post '/document/:app/:doc_id/title/:title' do |app, doc_id, title| begin - doc_key = DocKey.new app,doc_id - doc = Document.load( @storage_dir, doc_key ) + doc_key = DocKey.new app, doc_id + doc = Document.load(@storage_dir, doc_key) doc.title = title doc.save_metadata respond 200, 'Title updated' @@ -111,12 +112,14 @@ class App < Sinatra::Base # # POST params: # - callback_url - post '/document/:app/:doc_id/:version/:filename/:action' do |app,doc_id,version,filename,action| + post '/document/:app/:doc_id/:version/:filename/:action' do |app, doc_id, version, filename, action| begin doc_key = DocKey.new app, doc_id raise DocumentNotFound.new unless Document.exists? @storage_dir, doc_key + doc = Document.load @storage_dir, doc_key raise VersionNotFound.new unless doc.has_version? version + Sidekiq::ConversionWorker.perform_async doc_key, version, filename, action, params[:callback_url] respond 202, "Conversion initiated" rescue StandardError => e @@ -130,7 +133,7 @@ class App < Sinatra::Base # DELETE params: delete '/document/:app/:doc_id' do |app, doc_id| begin - Document.delete @storage_dir, DocKey.new(app,doc_id) + Document.delete @storage_dir, DocKey.new(app, doc_id) respond 200, 'Document deleted' rescue StandardError => e respond_with_error e @@ -143,7 +146,7 @@ class App < Sinatra::Base # DELETE params: delete '/document/:app/:doc_id/:version' do |app, doc_id, version| begin - doc = Document.load @storage_dir, DocKey.new(app,doc_id) + doc = Document.load @storage_dir, DocKey.new(app, doc_id) doc.delete_version version doc.save_metadata respond 200, 'Document version deleted' @@ -157,8 +160,8 @@ class App < Sinatra::Base # get '/document/:app/:doc_id/:version/:filename' do |app, doc_id, version, filename| begin - doc = Document.load @storage_dir, DocKey.new(app,doc_id) - ctype, file = doc.get_file( version, filename ) + doc = Document.load @storage_dir, DocKey.new(app, doc_id) + ctype, file = doc.get_file(version, filename) content_type ctype file rescue StandardError => e @@ -171,7 +174,7 @@ class App < Sinatra::Base # get '/document/:app/:doc_id' do |app, doc_id| begin - doc = Document.load @storage_dir, DocKey.new(app,doc_id) + doc = Document.load @storage_dir, DocKey.new(app, doc_id) respond 200, 'Information retrieved', doc.to_hash rescue StandardError => e respond_with_error e @@ -196,7 +199,7 @@ class App < Sinatra::Base end body = params[:file][:tempfile].read - content = Converter.new(logger:@logger).convert_file( params[:action], body, params[:language] ) + content = Converter.new(logger: @logger).convert_file(params[:action], body, params[:language]) content_type content.mime_type content rescue StandardError => e @@ -215,12 +218,12 @@ class App < Sinatra::Base post "/#{LegacyConverter::LEGACY}/convert" do begin body = if params[:file] - params[:file][:tempfile].read - elsif params[:url] - Net::HTTP.get URI(params[:url]) - else - raise DocumentNotFound.new "Please specify either 'file' or 'url' POST variable" - end + params[:file][:tempfile].read + elsif params[:url] + Net::HTTP.get URI(params[:url]) + else + raise DocumentNotFound.new "Please specify either 'file' or 'url' POST variable" + end path = LegacyConverter.new.convert_file params[:action], body, params[:language] converted_url = @legacy_url_base + path content_type 'application/json' @@ -252,13 +255,13 @@ class App < Sinatra::Base helpers do # Renders all responses (including errors) in a standard JSON format. - def respond status, message, extra={} + def respond status, message, extra = {} case status - when Colore::Error - status = status.http_code - when StandardError - extra[:backtrace] = status.backtrace if params[:backtrace] - status = 500 + when Colore::Error + status = status.http_code + when StandardError + extra[:backtrace] = status.backtrace if params[:backtrace] + status = 500 end content_type 'application/json' return status, { @@ -281,13 +284,13 @@ def respond_with_error error end # Renders all responses (including errors) in a standard JSON format. - def legacy_error status, message, extra={} + def legacy_error status, message, extra = {} case status - when Error - status = status.http_code - when StandardError - extra[:backtrace] = status.backtrace if params[:backtrace] - status = 500 + when Error + status = status.http_code + when StandardError + extra[:backtrace] = status.backtrace if params[:backtrace] + status = 500 end content_type 'application/json' return status, { diff --git a/lib/autoheathen/config.rb b/lib/autoheathen/config.rb index f2da303..1b9bf5b 100644 --- a/lib/autoheathen/config.rb +++ b/lib/autoheathen/config.rb @@ -1,6 +1,6 @@ module AutoHeathen module Config - def load_config defaults={}, config_file=nil, overwrites={} + def load_config defaults = {}, config_file = nil, overwrites = {} cfg = symbolize_keys(defaults) if config_file && File.exist?(config_file) cfg.merge! symbolize_keys(YAML::load_file config_file) @@ -10,7 +10,7 @@ def load_config defaults={}, config_file=nil, overwrites={} end def symbolize_keys(hash) - (hash||{}).inject({}){|result, (key, value)| + (hash || {}).inject({}) { |result, (key, value)| new_key = key.is_a?(String) ? key.to_sym : key new_value = value.is_a?(Hash) ? symbolize_keys(value) : value result[new_key] = new_value diff --git a/lib/autoheathen/email_processor.rb b/lib/autoheathen/email_processor.rb index 325ee3e..9574b6b 100644 --- a/lib/autoheathen/email_processor.rb +++ b/lib/autoheathen/email_processor.rb @@ -11,7 +11,7 @@ class EmailProcessor include AutoHeathen::Config # The only valid email headers we will allow forward to LEG_wikilex - ONWARD_HEADERS = ['Date','From','To','Subject','Content-Type','Content-Transfer-Encoding','Mime-Version'] + ONWARD_HEADERS = ['Date', 'From', 'To', 'Subject', 'Content-Type', 'Content-Transfer-Encoding', 'Mime-Version'] attr_reader :cfg, :logger @@ -27,8 +27,8 @@ class EmailProcessor # text_template: 'config/response.text.haml' Template for text part of response email (mode in [:return_to_sender,:email]) # html_template: 'config/response.html.haml' Template for HTML part of response email (ditto) # logger: nil Optional logger object - def initialize cfg={}, config_file=nil - @cfg = load_config( { # defaults + def initialize cfg = {}, config_file = nil + @cfg = load_config({ # defaults deliver: true, language: 'en', from: 'autoheathen', @@ -40,7 +40,7 @@ def initialize cfg={}, config_file=nil logger: nil, text_template: 'config/autoheathen.text.haml', html_template: 'config/autoheathen.html.haml', - }, config_file, cfg ) + }, config_file, cfg) @logger = @cfg[:logger] || Logger.new(nil) @logger.level = @cfg[:verbose] ? Logger::DEBUG : Logger::INFO end @@ -52,7 +52,7 @@ def process_rts email # Processes the given email, submits attachments to the Heathen server, delivers responses as configured # @param email [String] The encoded email (suitable to be decoded using Mail.read(input)) # @return [Hash] a hash of the decoded attachments (or the reason why they could not be decoded) - def process email, mail_to, is_rts=false + def process email, mail_to, is_rts = false documents = [] unless email.has_attachments? @@ -67,7 +67,7 @@ def process email, mail_to, is_rts=false # email.attachments.each do |attachment| begin - converter = Heathen::Converter.new( logger: logger ) + converter = Heathen::Converter.new(logger: logger) language = @cfg[:language] input_source = attachment.body.decoded action = get_action input_source.content_type @@ -114,9 +114,9 @@ def deliver_onward email, documents, mail_to # something weird goes on with Sharepoint, where the doc is dropped on the floor # so, remove any offending headers email.message_id = nil # make sure of message_id too - good_headers = ONWARD_HEADERS.map{ |h| h.downcase } + good_headers = ONWARD_HEADERS.map { |h| h.downcase } inspect_headers = email.header.map(&:name) - inspect_headers .each do |name| + inspect_headers.each do |name| unless good_headers.include? name.downcase email.header[name] = nil end @@ -142,17 +142,18 @@ def deliver_rts email, documents, mail_to mail.from @cfg[:from] mail.to mail_to # CCs to the original email will get a copy of the converted files as well - mail.cc (email.cc - email.to - (@cfg[:cc_blacklist]||[]) ) if email.cc # Prevent autoheathen infinite loop! + mail.cc (email.cc - email.to - (@cfg[:cc_blacklist] || [])) if email.cc # Prevent autoheathen infinite loop! # Don't prepend yet another Re: mail.subject "#{'Re: ' unless email.subject.start_with? 'Re:'}#{email.subject}" # Construct received path # TODO: is this in the right order? - #rcv = "by localhost(autoheathen); #{Time.now.strftime '%a, %d %b %Y %T %z'}" - #[email.received,rcv].flatten.each { |rec| mail.received rec.to_s } + # rcv = "by localhost(autoheathen); #{Time.now.strftime '%a, %d %b %Y %T %z'}" + # [email.received,rcv].flatten.each { |rec| mail.received rec.to_s } mail.return_path email.return_path if email.return_path mail.header['X-Received'] = email.header['X-Received'] if email.header['X-Received'] documents.each do |doc| next if doc[:content].nil? + mail.add_file filename: doc[:filename], content: doc[:content] end cfg = @cfg # stoopid Mail scoping @@ -206,12 +207,10 @@ def get_action content_type 'application/vnd.ms-powerpoint' => 'pdf', 'application/vnd.openxmlformats-officedocument.presentationml.presentation' => 'pdf', }[ct] - op = 'ocr' if ! op && ct.start_with?('image/') + op = 'ocr' if !op && ct.start_with?('image/') raise "Conversion from #{ct} is not supported" unless op + op end - end end - - diff --git a/lib/config.rb b/lib/config.rb index e0d977d..dc2038f 100644 --- a/lib/config.rb +++ b/lib/config.rb @@ -74,7 +74,7 @@ def self.config def self.method_missing *args if config.respond_to? args[0].to_sym - config.send( *args ) + config.send(*args) else super end diff --git a/lib/converter.rb b/lib/converter.rb index c2d42f5..5f11605 100644 --- a/lib/converter.rb +++ b/lib/converter.rb @@ -19,7 +19,7 @@ def initialize storage_dir: C_.storage_directory, logger: Logger.new(C_.conversi # @return the converted file name def convert doc_key, version, filename, action doc = Document.load @storage_dir, doc_key - ignore, orig_content = doc.get_file( version, filename) + ignore, orig_content = doc.get_file(version, filename) language = 'en' # TODO - add to spec and upload new_content = convert_file action, orig_content, language # TODO - handling for variant formats with the same extension @@ -36,12 +36,12 @@ def convert doc_key, version, filename, action # @param orig_content [String] the body of the file to convert # @param language [String] the file's language # @return [String] the converted file body - def convert_file action, orig_content, language='en' - Heathen::Converter.new(logger:@logger).convert(action, orig_content, language) + def convert_file action, orig_content, language = 'en' + Heathen::Converter.new(logger: @logger).convert(action, orig_content, language) rescue Heathen::TaskNotFound => e - raise InvalidAction.new( e.message ) + raise InvalidAction.new(e.message) rescue Heathen::Error => e - raise ConversionError.new( e ) + raise ConversionError.new(e) end end end diff --git a/lib/doc_key.rb b/lib/doc_key.rb index 0c1863d..f9491f9 100644 --- a/lib/doc_key.rb +++ b/lib/doc_key.rb @@ -41,7 +41,7 @@ def to_s # subdirectory names into which the docs can be distributed, rather than clumping # them into a single base directory (which will not scale well) def subdirectory - Digest::MD5.hexdigest(@doc_id)[0..(STORAGE_SUBDIRECTORY_LENGTH-1)] + Digest::MD5.hexdigest(@doc_id)[0..(STORAGE_SUBDIRECTORY_LENGTH - 1)] end private diff --git a/lib/document.rb b/lib/document.rb index a9a4cca..7417185 100644 --- a/lib/document.rb +++ b/lib/document.rb @@ -46,6 +46,7 @@ def self.exists? base_dir, doc_key def self.create base_dir, doc_key doc_dir = directory base_dir, doc_key raise DocumentExists.new if File.exist? doc_dir + FileUtils.mkdir_p doc_dir self.new base_dir, doc_key end @@ -56,6 +57,7 @@ def self.create base_dir, doc_key # @return [Document] def self.load base_dir, doc_key raise DocumentNotFound.new unless exists? base_dir, doc_key + doc = self.new base_dir, doc_key end @@ -65,7 +67,8 @@ def self.load base_dir, doc_key # @return [void]. def self.delete base_dir, doc_key return unless exists? base_dir, doc_key - FileUtils.rm_rf directory( base_dir, doc_key ) + + FileUtils.rm_rf directory(base_dir, doc_key) end # Constructor. @@ -83,20 +86,22 @@ def directory # @return the document title. def title - return '' unless File.exist?( directory + 'title' ) - File.read( directory + 'title' ).chomp + return '' unless File.exist?(directory + 'title') + + File.read(directory + 'title').chomp end # Sets the document title. def title= new_title return if new_title.to_s.empty? - File.open( directory + 'title', 'w' ) { |f| f.puts new_title } + + File.open(directory + 'title', 'w') { |f| f.puts new_title } end # Returns an array of the document version identifiers. def versions - versions = Dir.glob( directory + 'v*' ) - versions.reject { |v| ! v =~ /^v\d+$/ }.map{ |v| File.basename v }.sort + versions = Dir.glob(directory + 'v*') + versions.reject { |v| !v =~ /^v\d+$/ }.map { |v| File.basename v }.sort end # Returns true if the document has the specified version. @@ -111,8 +116,8 @@ def current_version # Returns the next version number (which would be created with [#new_version]). def next_version_number - v_no = (versions.last || 'v000').gsub(/v/,'').to_i + 1 - "v%03d"%[v_no] + v_no = (versions.last || 'v000').gsub(/v/, '').to_i + 1 + "v%03d" % [v_no] end # Creates a new version, ready to store documents in @@ -135,17 +140,19 @@ def new_version &block # @param body [String or IO] the file contents (binary string or IO) # @param author [String] the author of the file (optional) # @return [void] - def add_file version, filename, body, author=nil - raise VersionNotFound.new unless File.exist?( directory + version ) + def add_file version, filename, body, author = nil + raise VersionNotFound.new unless File.exist?(directory + version) + body = StringIO.new(body) unless body.respond_to?(:read) # string -> IO - File.open( directory + version + filename, "wb" ) { |f| IO.copy_stream(body,f) } - File.open( directory + version + AUTHOR_FILE, 'w' ) { |f| f.write author } + File.open(directory + version + filename, "wb") { |f| IO.copy_stream(body, f) } + File.open(directory + version + AUTHOR_FILE, 'w') { |f| f.write author } end # Sets the specified version as current. def set_current version - raise VersionNotFound.new unless File.exist?( directory + version ) + raise VersionNotFound.new unless File.exist?(directory + version) raise InvalidVersion.new unless version =~ /^v\d+$/ + # need to do this, or ln_s will put the symlink *into* the old dir File.unlink directory + CURRENT if File.exist? directory + CURRENT FileUtils.ln_s version, directory + CURRENT, force: true @@ -153,10 +160,11 @@ def set_current version # Deletes the given version, including its files. def delete_version version - return unless File.exist?( directory + version ) + return unless File.exist?(directory + version) raise VersionIsCurrent.new if version == CURRENT - raise VersionIsCurrent.new if (directory + CURRENT).realpath == (directory+version).realpath - FileUtils.rm_rf( directory + version ) + raise VersionIsCurrent.new if (directory + CURRENT).realpath == (directory + version).realpath + + FileUtils.rm_rf(directory + version) end # Returns the URL query path for the given file. This can be used to construct a @@ -178,6 +186,7 @@ def file_path version, filename def get_file version, filename path = directory + version + filename raise FileNotFound unless File.exist? path + body = File.read path return body.mime_type, body end @@ -190,14 +199,16 @@ def to_hash Dir.glob(directory + v + '*').each do |file| pfile = Pathname.new(file) next if pfile.basename.to_s == AUTHOR_FILE + content_type = File.read(pfile, [200, pfile.size].min).mime_type - author = File.read( pfile.parent + AUTHOR_FILE ).chomp rescue nil - suffix = pfile.extname.gsub( /\./, '') + author = File.read(pfile.parent + AUTHOR_FILE).chomp rescue nil + suffix = pfile.extname.gsub(/\./, '') next if suffix.empty? + v_list[v][suffix] = { content_type: content_type, filename: pfile.basename.to_s, - path: file_path(v,pfile.basename), + path: file_path(v, pfile.basename), author: author, created_at: pfile.mtime, } @@ -216,10 +227,9 @@ def to_hash # This metadata is just the {#to_hash}, as JSON, and is intended for access by client # applications. It is not used by Colore for anything. def save_metadata - File.open( directory + 'metadata.json', "w" ) do |f| + File.open(directory + 'metadata.json', "w") do |f| f.puts JSON.pretty_generate(to_hash) end end end - end diff --git a/lib/errors.rb b/lib/errors.rb index 87143f6..3346a1d 100644 --- a/lib/errors.rb +++ b/lib/errors.rb @@ -1,35 +1,45 @@ module Colore class Error < StandardError attr_accessor :http_code + def initialize http_code, message super message @http_code = http_code end end + class InvalidParameter < Error - def initialize param=nil; super 400, "Invalid parameter #{param}"; end + def initialize param = nil; super 400, "Invalid parameter #{param}"; end end + class DocumentExists < Error def initialize; super 409, 'A document with this doc_id already exists'; end end + class DocumentNotFound < Error def initialize; super 404, 'Document not found'; end end + class VersionNotFound < Error def initialize; super 400, 'Version not found'; end end + class InvalidVersion < Error def initialize; super 400, 'Invalid version name'; end end + class VersionIsCurrent < Error def initialize; super 400, 'Version is current, change current version first'; end end + class InvalidAction < Error - def initialize message=nil; super 400, (message||'Invalid action') ; end + def initialize message = nil; super 400, (message || 'Invalid action'); end end + class FileNotFound < Error def initialize; super 400, 'Requested file not found'; end end + class ConversionError < Error def initialize heathen_error super 500, "Conversion error: #{heathen_error.message}" diff --git a/lib/heathen.rb b/lib/heathen.rb index 799feb3..dcf19c2 100644 --- a/lib/heathen.rb +++ b/lib/heathen.rb @@ -9,6 +9,6 @@ require_relative 'heathen/converter' require_relative 'heathen/executioner' require_relative 'heathen/processor' -Dir.glob( (HEATHEN_BASE+'processor_methods'+'*.rb').to_s ).each do |method| +Dir.glob((HEATHEN_BASE + 'processor_methods' + '*.rb').to_s).each do |method| require_relative method end diff --git a/lib/heathen/converter.rb b/lib/heathen/converter.rb index b5981c4..cb0aae0 100644 --- a/lib/heathen/converter.rb +++ b/lib/heathen/converter.rb @@ -4,7 +4,7 @@ module Heathen # The Converter takes the given action and input content, identifies the task required # to perform the action, then constructs a [Processor] to convert the document. class Converter - def initialize( logger: Logger.new(nil) ) + def initialize(logger: Logger.new(nil)) @logger = logger end @@ -13,7 +13,7 @@ def initialize( logger: Logger.new(nil) ) # @param content [String] the document body to be converted # @param language [String] the document language (defaults to 'en') # @return [String] the converted document body - def convert action, content, language='en' + def convert action, content, language = 'en' job = Job.new action, content, language processor = Heathen::Processor.new job: job, logger: @logger begin diff --git a/lib/heathen/errors.rb b/lib/heathen/errors.rb index 6e7ecad..bdbc4e0 100644 --- a/lib/heathen/errors.rb +++ b/lib/heathen/errors.rb @@ -22,9 +22,9 @@ def initialize action, mime_type # Abstract step-wise error - the [#message] will also say in which method the error # occurred (So could be supplied to the client application without confusing anybody) class StepError < Error - def initialize method=nil + def initialize method = nil calling_method = caller[2] - method = calling_method.gsub(/.*[`](.*)'$/,'\\1') + method = calling_method.gsub(/.*[`](.*)'$/, '\\1') super "#{message} in step '#{method}'" end end @@ -38,22 +38,22 @@ def initialize expected, got # Raised if the task step was unable to perform the conversion class ConversionFailed < StepError - def initialize message=nil - super( message || "Conversion failed" ) + def initialize message = nil + super(message || "Conversion failed") end end # Raised if an input parameter to the task step was unrecognised or invalid class InvalidParameterInStep < StepError def initialize param_name, param_value - super( "Invalid parameter: #{param_name}: #{param_value}" ) + super("Invalid parameter: #{param_name}: #{param_value}") end end # Raised if an job language is invalid for the step class InvalidLanguageInStep < StepError def initialize language - super( "Invalid language: #{language}" ) + super("Invalid language: #{language}") end end end diff --git a/lib/heathen/executioner.rb b/lib/heathen/executioner.rb index f4df4de..56502d3 100644 --- a/lib/heathen/executioner.rb +++ b/lib/heathen/executioner.rb @@ -1,5 +1,4 @@ module Heathen - # An Executioner object will execute the given command, storing exit status, # STDOUT and STDERR for perusal. class Executioner @@ -39,7 +38,7 @@ def execute(*argv) logger.info "[#{pid}] stderr: '#@stderr'" unless @stderr.empty? @last_exit_status = status - @last_messages = {stdout: @stdout, stderr: @stderr} + @last_messages = { stdout: @stdout, stderr: @stderr } @last_command = command.join(' ') return status @@ -51,7 +50,6 @@ def execute(*argv) # stderr as Strings. # def _execute(*argv, options) - builder = java.lang.ProcessBuilder.new builder.command(argv) @@ -64,10 +62,10 @@ def _execute(*argv, options) # Dirty hack, works on UNIX only. pid = if process.is_a?(Java::JavaLang::UNIXProcess) - prop = process.get_class.get_declared_field('pid') - prop.set_accessible true - prop.get_int(process) - end + prop = process.get_class.get_declared_field('pid') + prop.set_accessible true + prop.get_int(process) + end logger.info "[#{pid}] spawn '#{argv.join(' ')}'" @@ -97,7 +95,7 @@ def _execute(*argv, options) def _execute(*argv, options) command = argv.shift - Open3.popen3(ENV, [ command, "heathen: #{command}" ], *argv, + Open3.popen3(ENV, [command, "heathen: #{command}"], *argv, :chdir => options[:dir] || Dir.getwd ) do |stdin, stdout, stderr, wait_thr| pid = wait_thr[:pid] @@ -131,9 +129,10 @@ def quartering(heretics) end protected - def slaughter guilty - execute(*guilty) - slaughter(@heretics.shift) unless @heretics.size.zero? - end + + def slaughter guilty + execute(*guilty) + slaughter(@heretics.shift) unless @heretics.size.zero? + end end end diff --git a/lib/heathen/filename.rb b/lib/heathen/filename.rb index 4d43dfc..fee0fbb 100644 --- a/lib/heathen/filename.rb +++ b/lib/heathen/filename.rb @@ -6,7 +6,7 @@ module Filename # @return [String] a new file name, with an appropriate extension def self.suggest filename, mime_type ext = File.extname(filename) - temp_file = filename[0..-(ext.length+1)] + temp_file = filename[0..-(ext.length + 1)] ext = MIME::Types[mime_type].first.preferred_extension rescue 'txt' # FIXME use a saner extension "#{temp_file}.#{ext}" end diff --git a/lib/heathen/job.rb b/lib/heathen/job.rb index f1ee116..d22d590 100644 --- a/lib/heathen/job.rb +++ b/lib/heathen/job.rb @@ -25,7 +25,7 @@ class Job # @param content [String] the file body # @param language [String] the file's language # @param sandbox_dir [String] sandbox directory for temporary files - def initialize action, content, language='en', sandbox_dir=nil + def initialize action, content, language = 'en', sandbox_dir = nil @action = action @language = language @original_content = content @@ -47,9 +47,9 @@ def content= content # steps which work only on files, rather than content in memory. The first time this # method is called, for a given step, a temporary file is created and the content is # written to it. This will persist until the content is changed. - def content_file suffix='' + def content_file suffix = '' @tempfile ||= begin - t = Tempfile.new ["heathen",suffix], @sandbox_dir + t = Tempfile.new ["heathen", suffix], @sandbox_dir t.binmode t.write @content t.close diff --git a/lib/heathen/processor.rb b/lib/heathen/processor.rb index 8b6d0e5..927608a 100644 --- a/lib/heathen/processor.rb +++ b/lib/heathen/processor.rb @@ -17,11 +17,11 @@ class Processor # @param job [Job] the job to be performed. # @param logger [Logger] an optional logger. # @param base_tmpdir [String] the base directory for all temporary (sandbox_dir) files - def initialize( job:, logger: Logger.new(STDOUT), base_tmpdir:'/tmp' ) + def initialize(job:, logger: Logger.new(STDOUT), base_tmpdir: '/tmp') @job = job @logger = logger @executioner = Heathen::Executioner.new(@logger) - @sandbox_dir = Dir.mktmpdir( "heathen", base_tmpdir.to_s ) + @sandbox_dir = Dir.mktmpdir("heathen", base_tmpdir.to_s) job.sandbox_dir = @sandbox_dir end @@ -29,7 +29,7 @@ def initialize( job:, logger: Logger.new(STDOUT), base_tmpdir:'/tmp' ) # @param pattern [String] a regex pattern, e.g. "image/.*" # This is a helper method for mixin methods. def expect_mime_type pattern - raise InvalidMimeTypeInStep.new(pattern,job.mime_type) unless job.mime_type =~ %r[#{pattern}] + raise InvalidMimeTypeInStep.new(pattern, job.mime_type) unless job.mime_type =~ %r[#{pattern}] end # Performs a sub-task, defined by action. See [Task] for details. @@ -44,8 +44,8 @@ def clean_up end # Creates a new temporary file in the sandbox - def temp_file_name prefix='', suffix='' - Dir::Tmpname.create( [prefix,suffix], @sandbox_dir ){} + def temp_file_name prefix = '', suffix = '' + Dir::Tmpname.create([prefix, suffix], @sandbox_dir) {} end def config_file name diff --git a/lib/heathen/processor_methods/convert_image.rb b/lib/heathen/processor_methods/convert_image.rb index 00af809..39e6540 100644 --- a/lib/heathen/processor_methods/convert_image.rb +++ b/lib/heathen/processor_methods/convert_image.rb @@ -9,12 +9,13 @@ def convert_image to: 'tiff', params: nil target_file = temp_file_name '', ".#{to.to_s}" executioner.execute( - *[ Colore::C_.convert_path, + *[Colore::C_.convert_path, params.split(/ +/), job.content_file, - target_file ].flatten + target_file].flatten ) raise ConversionFailed.new if executioner.last_exit_status != 0 + c = File.read(target_file) job.content = File.read(target_file) File.unlink(target_file) diff --git a/lib/heathen/processor_methods/libreoffice.rb b/lib/heathen/processor_methods/libreoffice.rb index d4255c9..736304f 100644 --- a/lib/heathen/processor_methods/libreoffice.rb +++ b/lib/heathen/processor_methods/libreoffice.rb @@ -10,7 +10,7 @@ class Processor # pdf - convert to PDF (any libre-office format) # ms - corresponding Microsoft format # oo - corresponding LibreOffice format - def libreoffice( format: ) + def libreoffice(format:) suffixes = { 'pdf' => { '.*' => 'pdf', @@ -40,8 +40,9 @@ def libreoffice( format: ) } raise InvalidParameterInStep.new('format', format) unless suffixes[format.to_s] + to_suffix = nil - suffixes[format.to_s].each do |k,v| + suffixes[format.to_s].each do |k, v| to_suffix = v if job.mime_type =~ /#{k}/ end raise InvalidMimeTypeInStep.new('(various document formats)', job.mime_type) unless to_suffix @@ -68,7 +69,7 @@ def libreoffice( format: ) ) unless File.exist? target_file - raise ConversionFailed.new("Cannot find converted file (looking for #{File.basename(target_file)})" ) + raise ConversionFailed.new("Cannot find converted file (looking for #{File.basename(target_file)})") end output = File.read(target_file) @@ -105,6 +106,5 @@ def try_dev_shm stat = File.stat(DEV_SHM_PATH) stat.directory? && stat.writable? end - end end diff --git a/lib/heathen/processor_methods/tesseract.rb b/lib/heathen/processor_methods/tesseract.rb index 91c2108..9422164 100644 --- a/lib/heathen/processor_methods/tesseract.rb +++ b/lib/heathen/processor_methods/tesseract.rb @@ -23,6 +23,7 @@ def tesseract format: nil format, ) raise ConversionFailed.new(executioner.last_messages) if executioner.last_exit_status != 0 + suffix = format ? format : 'txt' target_file = "#{target_file}.#{suffix}" job.content = File.read(target_file) diff --git a/lib/heathen/processor_methods/wkhtmltopdf.rb b/lib/heathen/processor_methods/wkhtmltopdf.rb index 27f3571..1d01e4a 100644 --- a/lib/heathen/processor_methods/wkhtmltopdf.rb +++ b/lib/heathen/processor_methods/wkhtmltopdf.rb @@ -14,6 +14,7 @@ def wkhtmltopdf target_file, ].flatten) raise ConversionFailed.new('PDF converter rejected the request') if executioner.last_exit_status != 0 + job.content = File.read(target_file) File.unlink(target_file) end @@ -26,32 +27,32 @@ def _wkhtmltopdf_options(content) attrs = html.xpath("//meta[starts-with(@name, 'colore')]").inject({}) do |h, meta| name = meta.attributes['name'].value.sub(/^colore/, '-') value = if meta.attributes.key?('content') - unless (v = meta.attributes['content'].value.strip).size.zero? - # Some wkhtmltopdf command line options such as `--custom-header KEY VALUE` allows for 2 values as params - # instead of the regular 1 param. In that case the meta content is split by `;` delimiter - # https://wkhtmltopdf.org/usage/wkhtmltopdf.txt - # - # Ex: - # Let's say we have a report in HTML that needs to be converted into PDF and the report contains some images that - # are not publicly served but they need credentials. We can let wkhtml2pdf know it needs to send authentication - # credentials via custom HTML meta headers like - # - # - # - # - # that will arrive to colore as: - # meta.attributes['content'].value = "X-AUTH-TOKEN;FooBarBaz" - # - # and here we are splitting it in 2 keywords so the Executioner will escape them indidually as if we pass them - # sepparated by ' ' Open3 will send them to wkhtml2pdf as 2 single param enclosed in '""' which will basically be ignored - v.split ';' - end - end + unless (v = meta.attributes['content'].value.strip).size.zero? + # Some wkhtmltopdf command line options such as `--custom-header KEY VALUE` allows for 2 values as params + # instead of the regular 1 param. In that case the meta content is split by `;` delimiter + # https://wkhtmltopdf.org/usage/wkhtmltopdf.txt + # + # Ex: + # Let's say we have a report in HTML that needs to be converted into PDF and the report contains some images that + # are not publicly served but they need credentials. We can let wkhtml2pdf know it needs to send authentication + # credentials via custom HTML meta headers like + # + # + # + # + # that will arrive to colore as: + # meta.attributes['content'].value = "X-AUTH-TOKEN;FooBarBaz" + # + # and here we are splitting it in 2 keywords so the Executioner will escape them indidually as if we pass them + # sepparated by ' ' Open3 will send them to wkhtml2pdf as 2 single param enclosed in '""' which will basically be ignored + v.split ';' + end + end h.merge(name => value) end - attrs.inject([]){ |d, (k,v)| d << [k, v] }.flatten.compact + attrs.inject([]) { |d, (k, v)| d << [k, v] }.flatten.compact end end end diff --git a/lib/heathen/task.rb b/lib/heathen/task.rb index 9a4eb79..3ad2bca 100644 --- a/lib/heathen/task.rb +++ b/lib/heathen/task.rb @@ -1,5 +1,5 @@ module Heathen - # A Heathen task is a block of ruby called (by [Converter] in the context of a + # A Heathen task is a block of ruby called (by [Converter] in the context of a # [Processor] object. This allows us to call the Processor methods in a relatively # concise (and hopefully understandable) way. # @@ -22,13 +22,13 @@ def tasks # Registers a code block to be run for the given action and mime type. def register action, mime_type_pattern, &block - tasks[task_key(action,mime_type_pattern)] = { action: action, mime_type_pattern: mime_type_pattern, proc: block } + tasks[task_key(action, mime_type_pattern)] = { action: action, mime_type_pattern: mime_type_pattern, proc: block } end # Finds a registered task suitable for the given action and mime type (note, the first # suitable one will be selected). def find action, mime_type - tasks.each do |k,task| + tasks.each do |k, task| if task[:action] == action && mime_type =~ %r[#{task[:mime_type_pattern]}] return task end @@ -38,9 +38,9 @@ def find action, mime_type # Deletes any tasks that are keyed for the given action (and optional mime_type pattern, as specified in register) # Used for testing purposes - def clear action, mime_type=nil + def clear action, mime_type = nil tasks.keys.each do |key| - tasks.delete key if key =~ %r[#{task_key(action,mime_type)}] + tasks.delete key if key =~ %r[#{task_key(action, mime_type)}] end end @@ -67,12 +67,12 @@ def task_key action, mime_type Heathen::Task.register 'pdf', '.*' do case job.mime_type - when %r[image/*] - perform_task 'ocr' - when %r[text/html] - wkhtmltopdf - else - libreoffice format: 'pdf' + when %r[image/*] + perform_task 'ocr' + when %r[text/html] + wkhtmltopdf + else + libreoffice format: 'pdf' end end @@ -86,14 +86,14 @@ def task_key action, mime_type Heathen::Task.register 'txt', '.*' do case job.mime_type - when %r[image/*] - perform_task 'ocr_text' - when %r[text/html] - htmltotext - when %r[application/pdf] - pdftotext - else - libreoffice format: 'txt' + when %r[image/*] + perform_task 'ocr_text' + when %r[text/html] + htmltotext + when %r[application/pdf] + pdftotext + else + libreoffice format: 'txt' end end diff --git a/lib/legacy_converter.rb b/lib/legacy_converter.rb index 6a42ba8..1421fa1 100644 --- a/lib/legacy_converter.rb +++ b/lib/legacy_converter.rb @@ -26,7 +26,7 @@ def initialize storage_dir = C_.storage_directory # @param orig_content [String] the body of the file to convert # @param language [String] the file's language # @return [String] the path to the converted file - def convert_file action, orig_content, language='en' + def convert_file action, orig_content, language = 'en' content = Heathen::Converter.new.convert(action, orig_content, language) filename = Digest::SHA2.hexdigest content store_file filename, content @@ -41,6 +41,7 @@ def store_file filename, content # Loads and returns a legacy converted file def get_file filename raise "File does not exists" unless (@legacy_dir + filename).file? + File.read(@legacy_dir + filename) end end diff --git a/lib/sidekiq_workers.rb b/lib/sidekiq_workers.rb index 7811617..e79a57f 100644 --- a/lib/sidekiq_workers.rb +++ b/lib/sidekiq_workers.rb @@ -23,7 +23,7 @@ class ConversionWorker # @param filename [String] the file to convert # @param action [String] the conversion to perform # @param callback_url [String] optional callback URL - def perform doc_key_str, version, filename, action, callback_url=nil + def perform doc_key_str, version, filename, action, callback_url = nil doc_key = DocKey.parse doc_key_str new_filename = Converter.new.convert doc_key, version, filename, action status = 200 @@ -53,7 +53,7 @@ def log_error error, doc_key, filename, action log << "\nbacktrace:" log << " " << error.backtrace.join("\n ") - log.split("\n").each {|line| errlog.error(line) } + log.split("\n").each { |line| errlog.error(line) } end end @@ -80,7 +80,7 @@ def perform doc_key_str, version, action, new_filename, callback_url, status, de doc_id: doc_key.doc_id, version: version, action: action, - path: (doc.file_path(version,new_filename) if status && status < 300), + path: (doc.file_path(version, new_filename) if status && status < 300), } RestClient.post callback_url, rsp_hash end @@ -90,7 +90,6 @@ def perform doc_key_str, version, action, new_filename, callback_url, status, de # apps using the legacy service will request the file shortly after posting the # original, so won't need it after then). class LegacyPurgeWorker - include ::Sidekiq::Worker sidekiq_options queue: :purge, retry: 0, backtrace: true @@ -104,9 +103,10 @@ def perform purge_seconds = (C_.legacy_purge_days || 1).to_f * 86400.0 dir = LegacyConverter.new.legacy_dir dir.each_entry do |file| - next if (dir+file).directory? - if Time.now - (dir+file).ctime > purge_seconds - (dir+file).unlink + next if (dir + file).directory? + + if Time.now - (dir + file).ctime > purge_seconds + (dir + file).unlink logger.debug "Deleted old legacy file: #{file}" end end diff --git a/lib/utils.rb b/lib/utils.rb index 85381b9..7345917 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -3,16 +3,16 @@ module Utils # Deep conversion of all hash keys to symbols. def self.symbolize_keys obj case obj - when Hash - h = {} - obj.each do |k,v| - h[k.to_sym] = symbolize_keys v - end - h - when Array - obj.map{ |o| symbolize_keys o } - else - obj + when Hash + h = {} + obj.each do |k, v| + h[k.to_sym] = symbolize_keys v + end + h + when Array + obj.map { |o| symbolize_keys o } + else + obj end end diff --git a/spec/heathen/convert_spec.rb b/spec/heathen/convert_spec.rb index be0003a..c0f2eb3 100644 --- a/spec/heathen/convert_spec.rb +++ b/spec/heathen/convert_spec.rb @@ -5,15 +5,16 @@ class Heathen::Processor def valid_step_1 job.content << ",step 1" end + def valid_step_2 job.content << ",step 2" end + def failing_step_1 raise "It failed" end end - describe Heathen::Converter do before do Heathen::Task.clear 'test' diff --git a/spec/heathen/filename_spec.rb b/spec/heathen/filename_spec.rb index dbb8492..44e69e0 100644 --- a/spec/heathen/filename_spec.rb +++ b/spec/heathen/filename_spec.rb @@ -34,6 +34,5 @@ described_class.suggest_in_new_dir '/opt/users/joe/src/foo.pdf', mime_type, '/opt/users', '/home' ).to eq '/home/joe/src/foo.txt' end - end end diff --git a/spec/lib/app_spec.rb b/spec/lib/app_spec.rb index 16e4545..d333d9d 100644 --- a/spec/lib/app_spec.rb +++ b/spec/lib/app_spec.rb @@ -6,7 +6,7 @@ let(:appname) { 'app' } let(:doc_id) { '12345' } let(:filename) { 'arglebargle.docx' } - let(:doc_key) { Colore::DocKey.new(app,doc_id) } + let(:doc_key) { Colore::DocKey.new(app, doc_id) } let(:new_doc_id) { '54321' } let(:invalid_doc_id) { 'foobar' } let(:storage_dir) { tmp_storage_dir } @@ -15,7 +15,7 @@ def show_backtrace response if response.status == 500 begin - puts JSON.pretty_generate( JSON.parse response.body ) + puts JSON.pretty_generate(JSON.parse response.body) rescue StandardError => e puts response.body end @@ -37,7 +37,7 @@ def show_backtrace response put "/document/#{appname}/#{new_doc_id}/#{filename}", { title: 'A title', file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), - actions: [ 'ocr', 'pdf' ], + actions: ['ocr', 'pdf'], author: author, backtrace: true } @@ -45,7 +45,7 @@ def show_backtrace response expect(last_response.status).to eq 201 expect(last_response.content_type).to eq 'application/json' expect(JSON.parse(last_response.body)).to match( - {"status"=>201, "description"=>"Document stored", "app"=>"app", "doc_id"=>"54321", "path"=>"/document/app/54321/current/arglebargle.docx"} + { "status" => 201, "description" => "Document stored", "app" => "app", "doc_id" => "54321", "path" => "/document/app/54321/current/arglebargle.docx" } ) expect(Colore::Sidekiq::ConversionWorker).to have_received(:perform_async).twice end @@ -53,7 +53,7 @@ def show_backtrace response put "/document/#{appname}/#{doc_id}/#{filename}", { title: 'A title', file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), - actions: [ 'ocr', 'pdf' ], + actions: ['ocr', 'pdf'], backtrace: true } show_backtrace last_response @@ -68,14 +68,14 @@ def show_backtrace response it 'runs' do post "/document/#{appname}/#{doc_id}/#{filename}", { file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), - actions: [ 'ocr', 'pdf' ], + actions: ['ocr', 'pdf'], backtrace: true } show_backtrace last_response expect(last_response.status).to eq 201 expect(last_response.content_type).to eq 'application/json' expect(JSON.parse(last_response.body)).to match( - {"status"=>201, "description"=>"Document stored", "app"=>"app", "doc_id"=>"12345", "path"=>"/document/app/12345/current/arglebargle.docx"} + { "status" => 201, "description" => "Document stored", "app" => "app", "doc_id" => "12345", "path" => "/document/app/12345/current/arglebargle.docx" } ) expect(Colore::Sidekiq::ConversionWorker).to have_received(:perform_async).twice end @@ -83,7 +83,7 @@ def show_backtrace response it 'fails if document does not exist' do post "/document/#{appname}/#{new_doc_id}/#{filename}", { file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), - actions: [ 'ocr', 'pdf' ], + actions: ['ocr', 'pdf'], author: author, backtrace: true } @@ -121,7 +121,7 @@ def show_backtrace response expect(last_response.status).to eq 202 expect(last_response.content_type).to eq 'application/json' expect(JSON.parse(last_response.body)).to match( - {"status"=>202, "description"=>"Conversion initiated"} + { "status" => 202, "description" => "Conversion initiated" } ) expect(Colore::Sidekiq::ConversionWorker).to have_received(:perform_async).once end @@ -156,7 +156,7 @@ def show_backtrace response expect(last_response.status).to eq 200 expect(last_response.content_type).to eq 'application/json' expect(JSON.parse(last_response.body)).to match( - {"status"=>200, "description"=>"Document deleted"} + { "status" => 200, "description" => "Document deleted" } ) end end @@ -170,7 +170,7 @@ def show_backtrace response expect(last_response.status).to eq 200 expect(last_response.content_type).to eq 'application/json' expect(JSON.parse(last_response.body)).to match( - {"status"=>200, "description"=>"Document version deleted"} + { "status" => 200, "description" => "Document version deleted" } ) end it 'fails if you try to delete current' do @@ -263,7 +263,7 @@ def show_backtrace response action: 'pdf', file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), } - expect(foo).to receive(:convert_file).with(params[:action],String,nil) { "%PDF-1.4" } + expect(foo).to receive(:convert_file).with(params[:action], String, nil) { "%PDF-1.4" } post "/convert", params expect(last_response.status).to eq 200 expect(last_response.content_type).to eq 'application/pdf; charset=us-ascii' @@ -294,7 +294,7 @@ def show_backtrace response action: 'pdf', file: Rack::Test::UploadedFile.new(__FILE__, 'application/ruby'), } - expect(foo).to receive(:convert_file).with(params[:action],String,nil) { 'foobar' } + expect(foo).to receive(:convert_file).with(params[:action], String, nil) { 'foobar' } post "/#{Colore::LegacyConverter::LEGACY}/convert", params expect(last_response.status).to eq 200 expect(last_response.content_type).to eq 'application/json' @@ -310,7 +310,7 @@ def show_backtrace response url: 'http://localhost/foo/bar', } expect(Net::HTTP).to receive(:get).with(URI(params[:url])) { 'The quick brown flox' } - expect(foo).to receive(:convert_file).with(params[:action],String,nil) { 'foobar' } + expect(foo).to receive(:convert_file).with(params[:action], String, nil) { 'foobar' } post "/#{Colore::LegacyConverter::LEGACY}/convert", params expect(last_response.status).to eq 200 expect(last_response.content_type).to eq 'application/json' diff --git a/spec/lib/autoheathen/config_spec.rb b/spec/lib/autoheathen/config_spec.rb index 32ec7d4..1f36502 100644 --- a/spec/lib/autoheathen/config_spec.rb +++ b/spec/lib/autoheathen/config_spec.rb @@ -18,26 +18,26 @@ it "loads config with no defaults" do cfg = @obj.load_config nil, nil, { 'cow' => 'overcow', :rat => 'overrat' } - expect( cfg ).to eq( { + expect(cfg).to eq({ cow: 'overcow', rat: 'overrat', - } ) + }) end it "loads config from all sources" do defaults = { 'foo' => 'fooble', :bar => 'barble', 'bob' => 'bobble', :cow => 'cowble', :rat => 'ratble' } - @tempfile.write( { + @tempfile.write({ 'bob' => 'filebob', 'roger' => 'fileroger', - }.to_yaml ) + }.to_yaml) @tempfile.close cfg = @obj.load_config defaults, @tempfile.path, { 'cow' => 'overcow', :rat => 'overrat' } - expect( cfg[:foo] ).to eq 'fooble' - expect( cfg[:bar] ).to eq 'barble' - expect( cfg[:bob] ).to eq 'filebob' - expect( cfg[:roger] ).to eq 'fileroger' - expect( cfg[:cow] ).to eq 'overcow' - expect( cfg[:rat] ).to eq 'overrat' + expect(cfg[:foo]).to eq 'fooble' + expect(cfg[:bar]).to eq 'barble' + expect(cfg[:bob]).to eq 'filebob' + expect(cfg[:roger]).to eq 'fileroger' + expect(cfg[:cow]).to eq 'overcow' + expect(cfg[:rat]).to eq 'overrat' end it "symbolizes keys" do @@ -47,18 +47,18 @@ 'horse' => { 'duck' => :duckle, 'fish' => 'fishle', - 'eagle' => [ 'the', 'quick', 'brown', 'fox' ] + 'eagle' => ['the', 'quick', 'brown', 'fox'] } } hash = @obj.symbolize_keys in_hash - expect( hash ).to eq( { + expect(hash).to eq({ dog: 'doggle', cat: 'cattle', horse: { duck: :duckle, fish: 'fishle', - eagle: [ 'the', 'quick', 'brown', 'fox' ] + eagle: ['the', 'quick', 'brown', 'fox'] } - } ) + }) end end diff --git a/spec/lib/autoheathen/email_processor_spec.rb b/spec/lib/autoheathen/email_processor_spec.rb index 195ce0e..e29fb2e 100644 --- a/spec/lib/autoheathen/email_processor_spec.rb +++ b/spec/lib/autoheathen/email_processor_spec.rb @@ -3,17 +3,17 @@ describe AutoHeathen::EmailProcessor do let(:processor) { - AutoHeathen::EmailProcessor.new( { - cc_blacklist: [ 'wikilex@ifad.org' ], - }, fixture('autoheathen/autoheathen.yml' ) ) + AutoHeathen::EmailProcessor.new({ + cc_blacklist: ['wikilex@ifad.org'], + }, fixture('autoheathen/autoheathen.yml')) } let!(:email_to) { 'bob@localhost.localdomain' } let(:email) { - m = Mail.read( fixture('autoheathen/test1.eml') ) - m.to [ email_to ] - m.from [ 'bob@deviant.localdomain' ] - m.cc [ 'mrgrumpy', 'marypoppins', email_to, 'wikilex@ifad.org' ] - m.return_path [ 'jblackman@debian.localdomain' ] + m = Mail.read(fixture('autoheathen/test1.eml')) + m.to [email_to] + m.from ['bob@deviant.localdomain'] + m.cc ['mrgrumpy', 'marypoppins', email_to, 'wikilex@ifad.org'] + m.return_path ['jblackman@debian.localdomain'] m.header['X-Received'] = 'misssilly' m } @@ -35,7 +35,7 @@ expect(mail.to).to eq [to_address] expect(mail.subject).to eq "Fwd: Convert: please" expect(mail.attachments.size).to eq 2 # includes unconvertible attachment - expect(mail.attachments.map(&:filename)).to match_array %w[ test1.doc quickfox.pdf ] + expect(mail.attachments.map(&:filename)).to match_array %w[test1.doc quickfox.pdf] expect(mail.delivery_method.settings[:port]).to eq 25 expect(mail.delivery_method.settings[:address]).to eq 'localhost' expect(mail.cc).to be_nil # non-rts should not forward converted docs to anybody else @@ -59,7 +59,7 @@ expect(mail.html_part.decoded.size).to be > 0 expect(mail.delivery_method.settings[:port]).to eq 25 expect(mail.delivery_method.settings[:address]).to eq 'localhost' - expect(mail.cc).to eq [ 'mrgrumpy', 'marypoppins' ] # Test to exclude email_to & blacklist + expect(mail.cc).to eq ['mrgrumpy', 'marypoppins'] # Test to exclude email_to & blacklist expect(mail.return_path).to eq 'jblackman@debian.localdomain' expect(mail.header['X-Received'].to_s).to eq 'misssilly' end @@ -82,7 +82,7 @@ expect(processor.get_action 'image/tiff').to eq 'ocr' expect(processor.get_action 'application/pdf; charset=utf-8').to eq 'ocr' expect(processor.get_action 'application/msword').to eq 'pdf' - expect{processor.get_action 'foobar'}.to raise_error(RuntimeError) + expect { processor.get_action 'foobar' }.to raise_error(RuntimeError) end it 'handles null subject' do diff --git a/spec/lib/converter_spec.rb b/spec/lib/converter_spec.rb index 3c66df1..284b718 100644 --- a/spec/lib/converter_spec.rb +++ b/spec/lib/converter_spec.rb @@ -2,7 +2,7 @@ describe Colore::Converter do let(:storage_dir) { tmp_storage_dir } - let(:doc_key) { Colore::DocKey.new('app','12345') } + let(:doc_key) { Colore::DocKey.new('app', '12345') } let(:version) { 'v001' } let(:filename) { 'arglebargle.docx' } let(:action) { 'pdf' } diff --git a/spec/lib/doc_key_spec.rb b/spec/lib/doc_key_spec.rb index ffd2896..3a03d96 100644 --- a/spec/lib/doc_key_spec.rb +++ b/spec/lib/doc_key_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Colore::DocKey do - let(:doc_key) { described_class.new('myapp','mydoc') } + let(:doc_key) { described_class.new('myapp', 'mydoc') } context '.initialize' do it 'throws error if app is invalid' do diff --git a/spec/lib/document_spec.rb b/spec/lib/document_spec.rb index 7e6e484..d4e29bd 100644 --- a/spec/lib/document_spec.rb +++ b/spec/lib/document_spec.rb @@ -4,8 +4,8 @@ describe Colore::Document do let(:app) { 'app' } let(:doc_id) { '12345' } - let(:doc_key) { Colore::DocKey.new(app,doc_id) } - let(:invalid_doc_key) { Colore::DocKey.new(app,'bollox') } + let(:doc_key) { Colore::DocKey.new(app, doc_id) } + let(:invalid_doc_key) { Colore::DocKey.new(app, 'bollox') } let(:storage_dir) { tmp_storage_dir } let(:document) { described_class.load storage_dir, doc_key } let(:author) { 'spliffy' } @@ -20,30 +20,30 @@ context '.directory' do it 'runs' do - expect(described_class.directory(storage_dir,doc_key).to_s).to_not be_nil + expect(described_class.directory(storage_dir, doc_key).to_s).to_not be_nil end end context '.exists?' do it 'runs' do - expect(described_class.exists?(storage_dir,doc_key)).to eq true + expect(described_class.exists?(storage_dir, doc_key)).to eq true end it 'returns false if directory does not exist' do - expect(described_class.exists?(storage_dir,invalid_doc_key)).to eq false + expect(described_class.exists?(storage_dir, invalid_doc_key)).to eq false end end context '.create' do it 'runs' do - create_key = Colore::DocKey.new('app2','foo') + create_key = Colore::DocKey.new('app2', 'foo') doc = described_class.create storage_dir, create_key expect(doc).to_not be_nil expect(described_class.exists?(storage_dir, create_key)).to eq true end it 'raises error if doc already exists' do - expect{ + expect { described_class.create storage_dir, doc_key }.to raise_error Colore::DocumentExists end @@ -56,7 +56,7 @@ end it 'raises exception if directory does not exist' do - expect{ + expect { described_class.load storage_dir, invalid_doc_key }.to raise_error Colore::DocumentNotFound end @@ -99,7 +99,7 @@ context '#versions' do it 'runs' do - expect(document.versions).to match_array [ 'v001', 'v002' ] + expect(document.versions).to match_array ['v001', 'v002'] end end @@ -150,7 +150,7 @@ document.add_file 'v002', File.basename(file), body, author expect(File.exists? document.directory + 'v002' + File.basename(file)).to eq true expect(File.exists? document.directory + 'v002' + described_class::AUTHOR_FILE).to eq true - expect(File.read( document.directory + 'v002' + described_class::AUTHOR_FILE).chomp ).to eq author + expect(File.read(document.directory + 'v002' + described_class::AUTHOR_FILE).chomp).to eq author end it 'runs with IO for body' do file = __FILE__ @@ -163,8 +163,8 @@ context '#set_current' do it 'runs' do document.set_current 'v001' - st1 = File.stat( document.directory + 'current' ) - st2 = File.stat( document.directory + 'v001' ) + st1 = File.stat(document.directory + 'current') + st2 = File.stat(document.directory + 'v001') expect(st1.ino).to eq st2.ino end @@ -232,10 +232,10 @@ context '#to_hash' do it 'runs' do - testhash = JSON.parse( File.read(fixture('document.json')) ) + testhash = JSON.parse(File.read(fixture('document.json'))) testhash = Colore::Utils.symbolize_keys testhash dochash = Colore::Utils.symbolize_keys document.to_hash - dochash[:versions].each do |k,v| + dochash[:versions].each do |k, v| v.each { |k1, v1| v1.delete :created_at } end expect(dochash).to match testhash diff --git a/spec/lib/sidekiq_workers_spec.rb b/spec/lib/sidekiq_workers_spec.rb index dee9c15..1085e22 100644 --- a/spec/lib/sidekiq_workers_spec.rb +++ b/spec/lib/sidekiq_workers_spec.rb @@ -2,7 +2,7 @@ require 'rest_client' describe Colore::Sidekiq::ConversionWorker do - let(:doc_key) { Colore::DocKey.new('app','12345') } + let(:doc_key) { Colore::DocKey.new('app', '12345') } let(:callback_url) { 'http://foo/bar' } before do @@ -18,7 +18,7 @@ end it 'gives up on Heathen::TaskNotFound' do - allow(@mock_converter).to receive(:convert) { raise Heathen::TaskNotFound.new('foo','bar') } + allow(@mock_converter).to receive(:convert) { raise Heathen::TaskNotFound.new('foo', 'bar') } expect(Colore::Sidekiq::CallbackWorker).to receive(:perform_async) {} described_class.new.perform doc_key.to_s, 'current', 'arglebargle.docx', 'pdf', callback_url end @@ -32,7 +32,7 @@ end describe Colore::Sidekiq::CallbackWorker do - let(:doc_key) { Colore::DocKey.new('app','12345') } + let(:doc_key) { Colore::DocKey.new('app', '12345') } let(:callback_url) { 'http://foo/bar' } before do setup_storage @@ -43,7 +43,7 @@ end context '#perform' do it 'runs' do - expect(RestClient).to receive(:post).with(callback_url,Hash) + expect(RestClient).to receive(:post).with(callback_url, Hash) described_class.new.perform doc_key.to_s, 'current', 'arglebargle.docx', 'pdf', callback_url, 250, 'foobar' end end diff --git a/spec/lib/utils_spec.rb b/spec/lib/utils_spec.rb index e3c2eac..6324a1c 100644 --- a/spec/lib/utils_spec.rb +++ b/spec/lib/utils_spec.rb @@ -12,12 +12,12 @@ 'house_number' => 12, 'street' => 'Foo st', city: 'Boston', - phone_numbers: [ 1234567, '1234567' ], + phone_numbers: [1234567, '1234567'], }, 'rank' => 'Constable', 'awards' => [ - { type: 'Medal of Honour', 'reason'=>'Stupidity' }, - { 'type'=>'George Cross', reason: 'Bravery' }, + { type: 'Medal of Honour', 'reason' => 'Stupidity' }, + { 'type' => 'George Cross', reason: 'Bravery' }, ] } expect(symbolize_keys h).to eq({ @@ -26,7 +26,7 @@ house_number: 12, street: 'Foo st', city: 'Boston', - phone_numbers: [ 1234567, '1234567' ], + phone_numbers: [1234567, '1234567'], }, rank: 'Constable', awards: [ @@ -37,7 +37,7 @@ end it 'symbolizes array' do - expect(symbolize_keys([1234,'fred'])).to match_array [1234,'fred'] + expect(symbolize_keys([1234, 'fred'])).to match_array [1234, 'fred'] end it 'symbolizes fixnum' do @@ -64,7 +64,6 @@ } } expect(symbolize_keys(h)).to eq h - end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 813397c..23e9c8d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -25,7 +25,7 @@ def spec_logger Logger.new('spec/output.log') end -Dir.glob( (SPEC_BASE+"helpers"+"**.rb").to_s ).each do |helper| +Dir.glob((SPEC_BASE + "helpers" + "**.rb").to_s).each do |helper| require helper end