diff --git a/Jenkinsfile b/Jenkinsfile index 467f7f6..ff87c5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,13 @@ pipeline { "JS Lint": { node(label: 'docker') { - sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' + script { + try { + sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' + } catch (err) { + echo "Unstable: ${err}" + } + } } }, @@ -132,7 +138,7 @@ pipeline { "Plone5": { node(label: 'docker') { - sh '''docker run -i --rm --name="$BUILD_TAG-plone5" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5 -v -vv -s $GIT_NAME''' + sh '''docker pull eeacms/plone-test:5; docker run -i --rm --name="$BUILD_TAG-plone5" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5 -v -vv -s $GIT_NAME''' } }, @@ -140,17 +146,7 @@ pipeline { node(label: 'docker') { sh '''docker run -i --rm --name="$BUILD_TAG-python3" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5-python3 -v -vv -s $GIT_NAME''' } - }, - - "Zope2": { - node(label: 'eea') { - script { - checkout scm - sh '''cd buildouts/zope2 && ./install.sh && ./bin/instance-uptest''' - } - } } - ) } } @@ -250,7 +246,7 @@ pipeline { withSonarQubeEnv('Sonarqube') { sh '''sed -i "s|/plone/instance/src/$GIT_NAME|$(pwd)|g" coverage.xml''' sh '''find xunit-functional -type f -exec mv {} xunit-reports/ ";"''' - sh "export PATH=$PATH:${scannerHome}/bin:${nodeJS}/bin; sonar-scanner -Dsonar.python.xunit.skipDetails=true -Dsonar.python.xunit.reportPath=xunit-reports/*.xml -Dsonar.python.coverage.reportPath=coverage.xml -Dsonar.sources=./eea -Dsonar.projectKey=$GIT_NAME-$BRANCH_NAME -Dsonar.projectVersion=$BRANCH_NAME-$BUILD_NUMBER" + sh "export PATH=$PATH:${scannerHome}/bin:${nodeJS}/bin; sonar-scanner -Dsonar.python.xunit.skipDetails=true -Dsonar.python.xunit.reportPath=xunit-reports/*.xml -Dsonar.python.coverage.reportPaths=coverage.xml -Dsonar.sources=./eea -Dsonar.projectKey=$GIT_NAME-$BRANCH_NAME -Dsonar.projectVersion=$BRANCH_NAME-$BUILD_NUMBER" sh '''try=2; while [ \$try -gt 0 ]; do curl -s -XPOST -u "${SONAR_AUTH_TOKEN}:" "${SONAR_HOST_URL}api/project_tags/set?project=${GIT_NAME}-${BRANCH_NAME}&tags=${SONARQUBE_TAGS},${BRANCH_NAME}" > set_tags_result; if [ \$(grep -ic error set_tags_result ) -eq 0 ]; then try=0; else cat set_tags_result; echo "... Will retry"; sleep 60; try=\$(( \$try - 1 )); fi; done''' } } diff --git a/README.rst b/README.rst index e894507..d847c14 100644 --- a/README.rst +++ b/README.rst @@ -54,7 +54,7 @@ Install
- + diff --git a/buildouts/zope2/.gitignore b/buildouts/zope2/.gitignore deleted file mode 100644 index 2d22114..0000000 --- a/buildouts/zope2/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -pip-selfcheck.json -.*.cfg -src -parts -*eggs -*-dev.yml -bootstrap* -bin -lib* -var -local -share -secret.cfg -include -.Python -.idea -uuid.txt diff --git a/buildouts/zope2/buildout.cfg b/buildouts/zope2/buildout.cfg deleted file mode 100644 index 955394e..0000000 --- a/buildouts/zope2/buildout.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[buildout] -extends = versions.cfg -package-name = eea.sentry -develop = ../../ diff --git a/buildouts/zope2/install.sh b/buildouts/zope2/install.sh deleted file mode 100755 index f71e583..0000000 --- a/buildouts/zope2/install.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -set -e - -red="\e[1;31m" -green="\e[1;32m" -blink="\e[1;31m" -NC="\e[0m" # No Color - -function info { - echo -e "${green}INFO: ${NC} $1" -} -function warn { - echo -e "${blink}WARN: ${NC} $1" -} -function error { - echo -e "${red}ERROR: ${NC} $1" -} - -CONFIG=$1 - -VERSION_CFG="versions.cfg" - -PIP=$(cat $VERSION_CFG | grep "^pip\s*=\s*" | sed 's/ *//g' | sed 's/^.*\=\s*//g') -SETUPTOOLS=$(cat $VERSION_CFG | grep "^setuptools\s*\=\s*" | sed 's/ *//g' | sed 's/=//g' | sed 's/[a-z]//g') -ZCBUILDOUT=$(cat $VERSION_CFG | grep "^zc\.buildout\s*=\s*" | sed 's/ *//g' | sed 's/^.*\=\s*//g') -WHEEL=$(cat $VERSION_CFG | grep "^wheel\s*=\s*" | sed 's/ *//g' | sed 's/^.*\=\s*//g') - -if [ -z "$CONFIG" ]; then - if [ -s "development.cfg" ]; then - CONFIG="development.cfg" - else - CONFIG="buildout.cfg" - fi -fi - -#echo -e "${green}" -info "Using $CONFIG" - -if [ -z "$PIP" ]; then - PIP="9.0.1" -fi - -info "Using pip $PIP" - -if [ -z "$SETUPTOOLS" ]; then - SETUPTOOLS="33.1.1" -fi - -info "Using setuptools $SETUPTOOLS" - -if [ -z "$ZCBUILDOUT" ]; then - ZCBUILDOUT="2.9.5" -fi - -info "Using zc.buildout $ZCBUILDOUT" - -if [ -z "$WHEEL" ]; then - WHEEL="0.29.0" -fi - -info "Using wheel $WHEEL" - -if [ -z "$PYTHON" ]; then - PYTHON="/usr/bin/env python2.7" -fi - -# Make sure python is 2.7 or later -PYTHON_OK=`$PYTHON -c 'import sys -print (sys.version_info >= (2, 7) and "1" or "0")'` - -if [ "$PYTHON_OK" = '0' ]; then - error "Python 2.7 or later is required" - error "EXAMPLE USAGE: PYTHON=/path/to/python2.7 ./install.sh" - exit 0 -fi - -info "Using Python: " -$($PYTHON --version) - -if [ -z "$VIRTUALENV" ]; then - VIRTUALENV="/usr/bin/env virtualenv" -fi - -info "Using virtualenv $VIRTUALENV" - - -info "Adding eggs directory" -mkdir -vp eggs - -if [ -s "bin/activate" ]; then - - warn "Already a virtualenv environment." - warn "Please remove bin/activate if you want to reinitialize it." - -else - - info "Running: $VIRTUALENV --clear ." - $VIRTUALENV --clear . - - info "Running: bin/pip install pip==$PIP setuptools==$SETUPTOOLS zc.buildout==$ZCBUILDOUT wheel==$WHEEL" - ./bin/pip install pip==$PIP setuptools==$SETUPTOOLS zc.buildout==$ZCBUILDOUT wheel==$WHEEL - -fi - -echo "" -echo "=========================================================================" -info "All set. Running ./bin/buildout -c $CONFIG" -echo "=========================================================================" -echo "" -exec bin/buildout -c $CONFIG diff --git a/buildouts/zope2/versions.cfg b/buildouts/zope2/versions.cfg deleted file mode 100644 index 91def22..0000000 --- a/buildouts/zope2/versions.cfg +++ /dev/null @@ -1,7 +0,0 @@ -[buildout] -extends = https://svn.eionet.europa.eu/repositories/Zope/trunk/www.eea.europa.eu/trunk/buildout-configs/latest-zope.cfg - -[versions] -plone.recipe.zope2instance = 4.3 -zc.buildout = 2.11.2 -setuptools = diff --git a/docs/HISTORY.txt b/docs/HISTORY.txt index bec3bc3..fe8e8c4 100644 --- a/docs/HISTORY.txt +++ b/docs/HISTORY.txt @@ -1,6 +1,11 @@ Changelog ========= +1.5 - (2021-09-24) +--------------------------- +* Change: Migrate to the new sentry SDK + [valipod] + 1.4 - (2019-12-02) --------------------------- * Bug fix: Fix JS integration diff --git a/eea/sentry/__init__.py b/eea/sentry/__init__.py index 03cece5..336c1e8 100644 --- a/eea/sentry/__init__.py +++ b/eea/sentry/__init__.py @@ -1,15 +1,26 @@ """ Main product initializer """ import os +import re +import sys import six import logging from contextlib import closing -from raven.contrib.zope import ZopeSentryHandler +from AccessControl.SecurityManagement import getSecurityManager +from AccessControl.users import nobody from zope.i18nmessageid.message import MessageFactory +from zope.globalrequest import getRequest if six.PY2: from eventlet.green import urllib2 as request else: from eventlet.green.urllib import request +from zope.component import adapter + +from ZPublisher.HTTPRequest import _filterPasswordFields +from ZPublisher.interfaces import IPubFailure +import sentry_sdk +from sentry_sdk.integrations.logging import ignore_logger +from eea.sentry.browser.sentry import get_site EEAMessageFactory = MessageFactory('eea') logger = logging.getLogger() @@ -28,19 +39,166 @@ def environment(): return env -def initialize(context): - """Initializer called when used as a Zope 2 product. +def _get_user_from_request(request): + user = request.get("AUTHENTICATED_USER", None) + + if user is None: + user = getSecurityManager().getUser() + + if user is not None and user != nobody: + user_dict = { + "id": user.getId(), + "email": user.getProperty("email") or "", + } + else: + user_dict = {'id': 'Anonymous'} + + return user_dict + + +def _get_other_from_request(request): + other = {} + for k, v in _filterPasswordFields(request.other.items()): + if k in ("PARENTS", "RESPONSE"): + continue + other[k] = repr(v) + return other + + +def _get_lazyitems_from_request(request): + lazy_items = {} + for k, v in _filterPasswordFields(request._lazies.items()): + lazy_items[k] = repr(v) + return lazy_items + + +def _get_form_from_request(request): + form = {} + for k, v in _filterPasswordFields(request.form.items()): + form[k] = repr(v) + return form + + +def _get_request_from_request(request): + # ensure that all header key-value pairs are strings + headers = dict() + for k, v in request.environ.items(): + if not isinstance(v, str): + v = str(v) + headers[k] = v + + body_pos = request.stdin.tell() + request.stdin.seek(0) + request.stdin.seek(body_pos) + http = dict( + headers=headers, + url=request.getURL(), + method=request.method, + host=request.environ.get("REMOTE_ADDR", ""), + ) + if "HTTP_USER_AGENT" in http["headers"]: + if "User-Agent" not in http["headers"]: + http["headers"]["User-Agent"] = http["headers"]["HTTP_USER_AGENT"] + if "QUERY_STRING" in http["headers"]: + http["query_string"] = http["headers"]["QUERY_STRING"] + + return http + + +def _before_send(event, hint): + """ + Inject Plone/Zope specific information (based on raven.contrib.zope) """ + request = getRequest() + + if request: + # We have no request if event is captured by errorRaisedSubscriber + # (see below) so extra information must be set there. + # If the event is send by pythons logging module + # we set extra info here. + if "other" not in event["extra"]: + event["extra"]["other"] = _get_other_from_request(request) + if "lazy items" not in event["extra"]: + event["extra"]["lazy items"] = _get_lazyitems_from_request(request) + if "form" not in event["extra"]: + event["extra"]["form"] = _get_form_from_request(request) + if "request" not in event["extra"]: + event["extra"]["request"] = _get_request_from_request(request) + + return event + + +def _get_browser_from_request(request): + ''' return browser and version as a tuple ''' + browsers = {'MSIE': 'Internet Explorer', 'OPR': 'Opera', + 'Trident': 'Internet Explorer', 'Edg': 'Edge'} + user_agent = request.environ['HTTP_USER_AGENT'] + for browser in ['Edg', 'Firefox', 'Seamonkey', 'OPR', 'Opera', 'Trident', + 'MSIE', 'Chrome', 'Chromium', 'Safari']: + match = re.findall(browser + '[/ ]?([0-9.]+)', user_agent) + if match: + return (browsers.get(browser, browser), match[0]) + + +def before_send(event, hint): + try: + return _before_send(event, hint) + except KeyError: + logger.warn("Could not extract data from request", exc_info=True) + + +def initialize(context): + """ Initializer """ + sentry_dsn = os.environ.get('SENTRY_DSN', '') if not sentry_dsn: return - - sentry_handler = ZopeSentryHandler(sentry_dsn, - site=os.environ.get('SENTRY_SITE', - os.environ.get('SERVER_NAME', "dev")), + sentry_sdk.init( + sentry_dsn, + max_breadcrumbs=50, + before_send=before_send, release=os.environ.get('SENTRY_RELEASE', - os.environ.get('EEA_KGS_VERSION', 'dev')), + os.environ.get('EEA_KGS_VERSION', 'dev')), environment=os.environ.get("SENTRY_ENVIRONMENT", environment()), - processors=['eea.sentry.processors.SanitizeZopeProcessor'], + traces_sample_rate=1.0, ) - logger.addHandler(sentry_handler) + logger.info('Sentry integration enabled') + ignore_logger("Zope.SiteErrorLog") + + +@adapter(IPubFailure) +def errorRaisedSubscriber(event): + exc_info = ( + sys.exc_info() + ) # Save exc_info before new exceptions (CannotGetPortalError) arise + portal = get_site(event.request) + try: + error_log = portal.error_log + except AttributeError: + error_log = None + + if error_log and exc_info[0].__name__ in error_log._ignored_exceptions: + return + + with sentry_sdk.push_scope() as scope: + scope.set_extra("other", _get_other_from_request(event.request)) + scope.set_extra("lazy items", + _get_lazyitems_from_request(event.request)) + scope.set_extra("form", _get_form_from_request(event.request)) + scope.set_extra("request", _get_request_from_request(event.request)) + user_info = _get_user_from_request(event.request) + if user_info and "id" in user_info: + scope.user = user_info + if portal: + site_id = portal.getId() + else: + site_id = os.environ.get('SENTRY_SITE', + os.environ.get('SERVER_NAME', "dev")) + scope.set_tag('site', site_id) + browser = _get_browser_from_request(event.request) + if browser: + scope.set_tag('browser', '%s %s' % browser) + scope.set_tag('browser.name', browser[0]) + scope.set_tag('browser.version', browser[1]) + + sentry_sdk.capture_exception(exc_info) \ No newline at end of file diff --git a/eea/sentry/browser/configure.zcml b/eea/sentry/browser/configure.zcml index 1bfb8bb..0e6c02b 100644 --- a/eea/sentry/browser/configure.zcml +++ b/eea/sentry/browser/configure.zcml @@ -4,7 +4,7 @@ xmlns:browser="http://namespaces.zope.org/browser" i18n_domain="eea"> - eval")>-1;r&&(b=k.exec(c[3]))?(c[3]=b[1],c[4]=b[2],c[5]=null):0!==o||c[5]||"undefined"==typeof a.columnNumber||(n[0].column=a.columnNumber+1),f={url:c[3],func:c[1]||j,args:c[2]?c[2].split(","):[],line:c[4]?+c[4]:null,column:c[5]?+c[5]:null}}if(!f.func&&f.line&&(f.func=j),f.url&&"blob:"===f.url.substr(0,5)){var s=new XMLHttpRequest;if(s.open("GET",f.url,!1),s.send(null),200===s.status){var t=s.responseText||"";t=t.slice(-300);var u=t.match(/\/\/# sourceMappingURL=(.*)$/);if(u){var v=u[1];"~"===v.charAt(0)&&(v=e()+v.slice(1)),f.url=v.slice(0,-4)}}}n.push(f)}return n.length?{name:a.name,message:a.message,url:d(),stack:n}:null}}function b(a,b,c,d){var e={url:b,line:c};if(e.url&&e.line){if(a.incomplete=!1,e.func||(e.func=j),a.stack.length>0&&a.stack[0].url===e.url){if(a.stack[0].line===e.line)return!1;if(!a.stack[0].line&&a.stack[0].func===e.func)return a.stack[0].line=e.line,
-!1}return a.stack.unshift(e),a.partial=!0,!0}return a.incomplete=!0,!1}function c(a,e){for(var h,i,k=/function\s+([_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*)?\s*\(/i,l=[],m={},n=!1,o=c.caller;o&&!n;o=o.caller)if(o!==f&&o!==g.report){if(i={url:null,func:j,line:null,column:null},o.name?i.func=o.name:(h=k.exec(o.toString()))&&(i.func=h[1]),"undefined"==typeof i.func)try{i.func=h.input.substring(0,h.input.indexOf("{"))}catch(p){}m[""+o]?n=!0:m[""+o]=!0,l.push(i)}e&&l.splice(0,e);var q={name:a.name,message:a.message,url:d(),stack:l};return b(q,a.sourceURL||a.fileName,a.line||a.lineNumber,a.message||a.description),q}function f(b,e){var f=null;e=null==e?0:+e;try{if(f=a(b))return f}catch(h){if(g.debug)throw h}try{if(f=c(b,e+1))return f}catch(h){if(g.debug)throw h}return{name:b.name,message:b.message,url:d()}}return f.augmentStackTraceWithInitialElement=b,f.computeStackTraceFromStackProp=a,f}(),b.exports=g}).call(this,"undefined"!=typeof global?global:"undefined"!=typeof self?self:"undefined"!=typeof window?window:{})},{5:5}],7:[function(a,b,c){function d(a,b){for(var c=0;c