From 11791c54ac7a0227f13f12f5dd256e2c1cab152e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Labb=C3=A9?= <slabqc@gmail.com>
Date: Wed, 12 Dec 2018 23:19:21 +0100
Subject: [PATCH 1/4] 25501: fixing the mma integrator

---
 src/sage/symbolic/integration/external.py | 318 +++++++++++++++++-----
 src/sage/symbolic/integration/integral.py |   2 +-
 2 files changed, 254 insertions(+), 66 deletions(-)

diff --git a/src/sage/symbolic/integration/external.py b/src/sage/symbolic/integration/external.py
index d938a29e6ba..6b9fdc046c2 100644
--- a/src/sage/symbolic/integration/external.py
+++ b/src/sage/symbolic/integration/external.py
@@ -65,6 +65,11 @@ def mma_free_integrator(expression, v, a=None, b=None):
         sage: mma_free_integrator(sin(x), x) # optional - internet
         -cos(x)
 
+    A definite integral::
+
+        sage: mma_free_integrator(e^(-x), x, a=0, b=oo) # optional - internet
+        1
+
     TESTS:
 
     Check that :trac:`18212` is resolved::
@@ -72,78 +77,261 @@ def mma_free_integrator(expression, v, a=None, b=None):
         sage: var('y')   # optional - internet
         y
         sage: integral(sin(y)^2, y, algorithm='mathematica_free') # optional - internet
-        -1/2*cos(y)*sin(y) + 1/2*y
+        1/2*y - 1/4*sin(2*y)
+
+    ::
 
         sage: mma_free_integrator(exp(-x^2)*log(x), x) # optional - internet
         1/2*sqrt(pi)*erf(x)*log(x) - x*hypergeometric((1/2, 1/2), (3/2, 3/2), -x^2)
+
+    """
+    math_expr = expression._mathematica_init_()
+    variable = v._mathematica_init_()
+    if a is None and b is None:
+        input = "Integrate[{},{}]".format(math_expr, variable)
+    elif a is not None and b is not None:
+        input = "Integrate[{},{{{},{},{}}}]".format(math_expr, variable,
+                    a._mathematica_init_(), b._mathematica_init_())
+    else:
+        raise ValueError('a(={}) and b(={}) should be both None'
+                         ' or both defined'.format(a,b))
+    json_page_data = request_wolfram_alpha(input)
+    all_outputs = parse_moutput_from_json(json_page_data)
+    if not all_outputs:
+        raise ValueError("no outputs found in the answer from Wolfram Alpha")
+    first_output = all_outputs[0]
+    return symbolic_expression_from_mathematica_string(first_output)
+
+def request_wolfram_alpha(input, verbose=False):
+    r"""
+    Request Wolfram Alpha website.
+
+    INPUT:
+
+    - ``input`` -- string
+    - ``verbose`` -- bool (default: ``False``)
+
+    OUTPUT:
+
+    json
+
+    EXAMPLES::
+
+        sage: from sage.symbolic.integration.external import request_wolfram_alpha
+        sage: page_data = request_wolfram_alpha('integrate Sin[x]') # optional internet
+        sage: page_data.keys()                                      # optional internet
+        [u'queryresult']
+        sage: page_data['queryresult'].keys()                       # optional internet
+        [u'sponsorCategories',
+         u'parsetimedout',
+         u'success',
+         u'timedoutpods',
+         u'parsetiming',
+         u'id',
+         u'timedout',
+         u'related',
+         u'numpods',
+         u'host',
+         u'version',
+         u'encryptedParsedExpression',
+         u'encryptedEvaluatedExpression',
+         u'error',
+         u'timing',
+         u'datatypes',
+         u'server',
+         u'pods',
+         u'recalculate']
+
     """
-    import re
     # import compatible with py2 and py3
-    from six.moves.urllib.request import urlopen
     from six.moves.urllib.parse import urlencode
-    # We need to integrate against x
-    vars = [str(x) for x in expression.variables()]
-    if any(len(x)>1 for x in vars):
-        raise NotImplementedError("Mathematica online integrator can only handle single letter variables.")
-    x = SR.var('x')
-    if repr(v) != 'x':
-        for i in range(ord('a'), ord('z')+1):
-            if chr(i) not in vars:
-                shadow_x = SR.var(chr(i))
-                break
-        expression = expression.subs({x:shadow_x}).subs({v: x})
-    params = urlencode({'expr': expression._mathematica_init_(), 'random': 'false'})
-    page = urlopen("http://integrals.wolfram.com/home.jsp", params).read()
-    page = page[page.index('"inputForm"'):page.index('"outputForm"')]
-    page = re.sub(r"\s", "", page)
-    mexpr = re.match(r".*Integrate.*==</em><br/>(.*)</p>", page).groups()[0]
-    try:
-        from sage.libs.pynac.pynac import symbol_table
-        from sage.interfaces.mathematica import _un_camel as un_camel
-        from sage.symbolic.constants import constants_name_table as constants
-        from sage.calculus.calculus import symbolic_expression_from_string
-        from sage.calculus.calculus import _find_func as find_func
-
-        expr = mexpr.replace('\n',' ').replace('\r', '')
-        expr = expr.replace('[', '(').replace(']', ')')
-        expr = expr.replace('{', '[').replace('}', ']')
-        lsymbols = symbol_table['mathematica'].copy()
-        autotrans = [str.lower,      # Try it in lower case
-                     un_camel,      # Convert `CamelCase` to `camel_case`
-                     lambda x: x     # Try the original name
-                    ]
-        # Find the MMA funcs/vars/constants - they start with a letter.
-        # Exclude exponents (e.g. 'e8' from 4.e8)
-        p = re.compile(r'(?<!\.)[a-zA-Z]\w*')
-
-        for m in p.finditer(expr):
-            # If the function, variable or constant is already in the
-            # translation dictionary, then just move on.
-            if m.group() in lsymbols:
-                pass
-            # Now try to translate all other functions -- try each strategy
-            # in `autotrans` and check if the function exists in Sage
-            elif m.end() < len(expr) and expr[m.end()] == '(':
-                for t in autotrans:
-                    f = find_func(t(m.group()), create_when_missing = False)
-                    if f is not None:
-                        lsymbols[m.group()] = f
-                        break
-                else:
-                    raise NotImplementedError("Don't know a Sage equivalent for Mathematica function '%s'." % m.group())
-            # Check if Sage has an equivalent constant
-            else:
-                for t in autotrans:
-                    if t(m.group()) in constants:
-                        lsymbols[m.group()] = constants[t(m.group())]
-                        break
-        ans = symbolic_expression_from_string(expr, lsymbols, accept_sequence=True)
-        if repr(v) != 'x':
-            ans = ans.subs({x:v}).subs({shadow_x:x})
-        return ans
-    except TypeError:
-        raise ValueError("Unable to parse: %s" % mexpr)
+    from six.moves.urllib.request import Request, build_opener, HTTPCookieProcessor
+    import json
+    from http.cookiejar import CookieJar
+
+    # we need cookies for this...
+    cj = CookieJar()
+    opener = build_opener(HTTPCookieProcessor(cj))
+    # build initial query for code
+    req = Request("http://www.wolframalpha.com/input/api/v1/code")
+    resp = opener.open(req)
+    # the website returns JSON containing the code
+    page_data = json.loads(resp.read().decode("utf-8"))
+    if not ("code" in page_data.keys()):
+        raise ValueError("Wolfram did not return a code")
+    proxy_code = page_data['code']
+    if verbose:
+        print("Code: {}".format(proxy_code))
+        print("Cookies: {}".format(cj))
+    # now we can make a request
+    # some parameters documented here:
+    #   https://products.wolframalpha.com/api/documentation/#parameter-reference
+    # the following are the parameters used by the website
+    params = {
+        'assumptionsversion': '2',
+        'async': 'true',
+        'banners': 'raw',
+        'debuggingdata': 'false',
+        'format': 'image,plaintext,imagemap,sound,minput,moutput',
+        'formattimeout': '8',
+        'input': input,
+        'output': 'JSON',
+        'parsetimeout': '5',
+        'podinfosasync': 'true',
+        'proxycode': proxy_code,
+        'recalcscheme': 'parallel',
+        'sbsdetails': 'true',
+        'scantimeout': '0.5',
+        'sponsorcategories': 'true',
+        'statemethod': 'deploybutton',
+        'storesubpodexprs': 'true'
+        }
+    # # we can also change some parameters
+    # params = {
+    #     'assumptionsversion': '2',
+    #     'banners': 'raw',
+    #     'format': 'minput,moutput',
+    #     'formattimeout': '8',
+    #     'input': input,
+    #     'output': 'JSON',
+    #     'parsetimeout': '5',
+    #     'proxycode': proxy_code,
+    #     'scantimeout': '0.5',
+    #     'storesubpodexprs': 'true'
+    # }
+    params = urlencode(params)
+    url = "https://www.wolframalpha.com/input/json.jsp?%s" % params
+    req = Request(url)
+    req.add_header('Referer', "https://www.wolframalpha.com/input/") # seems important
+    resp = opener.open(req)
+    # the website returns JSON containing the code
+    return json.loads(resp.read().decode("utf-8"))
+
+def parse_moutput_from_json(page_data, verbose=False):
+    r"""
+    Return the list of outputs found in the json (with key ``'moutput'``)
+
+    INPUT:
+
+    - ``page_data`` -- json obtained from Wolfram Alpha
+    - ``verbose`` -- bool (default: ``False``)
+
+    OUTPUT:
+
+    list of unicode strings
 
+    EXAMPLES::
+
+        sage: from sage.symbolic.integration.external import request_wolfram_alpha
+        sage: from sage.symbolic.integration.external import parse_moutput_from_json
+        sage: page_data = request_wolfram_alpha('integrate Sin[x]') # optional internet
+        sage: parse_moutput_from_json(page_data)                    # optional internet
+        [u'-Cos[x]']
+
+    ::
+
+        sage: page_data = request_wolfram_alpha('Sin[x]')           # optional internet
+        sage: L = parse_moutput_from_json(page_data)                # optional internet
+        sage: sorted(L)                                             # optional internet
+        [u'-Cos[x]', u'{{x == Pi C[1], Element[C[1], Integers]}}']
+
+    TESTS::
+
+        sage: page_data = request_wolfram_alpha('Integrate(Sin[z], y)')
+        sage: parse_moutput_from_json(page_data)
+        Traceback (most recent call last):
+        ...
+        ValueError: asking wolframalpha.com was not successful
+
+    """
+    queryresult = page_data['queryresult']
+    if not queryresult['success']:
+        raise ValueError('asking wolframalpha.com was not successful')
+    if not 'pods' in queryresult:
+        raise ValueError('json object contains no pods')
+    pods = queryresult['pods']
+    if verbose:
+        print("  Query successful: {}".format(queryresult['success']))
+        print("  Number of results: {}".format(len(pods)))
+    L = []
+    for i, result in enumerate(pods):
+        if verbose:
+            print("  Result #{}".format(i))
+            print("    Title: {}".format(result['title']))
+        if 'subpods' not in result:
+            continue
+        subpods = result[u'subpods']
+        for j, subpod in enumerate(subpods):
+            if verbose:
+                print("    Subpod #{}".format(j))
+                if 'minput' in subpod.keys():
+                    print("      MInput: {}".format(subpod['minput']))
+                if 'moutput' in subpod.keys():
+                    print("      MOutput: {}".format(subpod['moutput']))
+            if 'moutput' in subpod.keys():
+                L.append(subpod['moutput'])
+    return L
+
+def symbolic_expression_from_mathematica_string(mexpr):
+    r"""
+    Translate a mathematica string into a symbolic expression
+
+    INPUT:
+
+    - ``mexpr`` -- string
+
+    OUTPUT:
+
+    symbolic expression
+
+    EXAMPLES::
+
+        sage: from sage.symbolic.integration.external import symbolic_expression_from_mathematica_string
+        sage: symbolic_expression_from_mathematica_string(u'-Cos[x]')
+        -cos(x)
+
+    """
+    import re
+    from sage.libs.pynac.pynac import symbol_table
+    from sage.interfaces.mathematica import _un_camel as un_camel
+    from sage.symbolic.constants import constants_name_table as constants
+    from sage.calculus.calculus import symbolic_expression_from_string
+    from sage.calculus.calculus import _find_func as find_func
+
+    expr = mexpr.replace('\n',' ').replace('\r', '')
+    expr = expr.replace('[', '(').replace(']', ')')
+    expr = expr.replace('{', '[').replace('}', ']')
+    lsymbols = symbol_table['mathematica'].copy()
+    autotrans = [unicode.lower,      # Try it in lower case
+                un_camel,      # Convert `CamelCase` to `camel_case`
+                lambda x: x     # Try the original name
+                ]
+    # Find the MMA funcs/vars/constants - they start with a letter.
+    # Exclude exponents (e.g. 'e8' from 4.e8)
+    p = re.compile(r'(?<!\.)[a-zA-Z]\w*')
+
+    for m in p.finditer(expr):
+        # If the function, variable or constant is already in the
+        # translation dictionary, then just move on.
+        if m.group() in lsymbols:
+            pass
+        # Now try to translate all other functions -- try each strategy
+        # in `autotrans` and check if the function exists in Sage
+        elif m.end() < len(expr) and expr[m.end()] == '(':
+            for t in autotrans:
+                f = find_func(t(m.group()), create_when_missing = False)
+                if f is not None:
+                    lsymbols[m.group()] = f
+                    break
+            else:
+                raise NotImplementedError("Don't know a Sage equivalent for Mathematica function '%s'." % m.group())
+        # Check if Sage has an equivalent constant
+        else:
+            for t in autotrans:
+                if t(m.group()) in constants:
+                    lsymbols[m.group()] = constants[t(m.group())]
+                    break
+    return symbolic_expression_from_string(expr, lsymbols, accept_sequence=True)
 
 def fricas_integrator(expression, v, a=None, b=None, noPole=True):
     """
diff --git a/src/sage/symbolic/integration/integral.py b/src/sage/symbolic/integration/integral.py
index 86357a0ea10..d400c89670d 100644
--- a/src/sage/symbolic/integration/integral.py
+++ b/src/sage/symbolic/integration/integral.py
@@ -509,7 +509,7 @@ def integrate(expression, v=None, a=None, b=None, algorithm=None, hold=False):
         sage: _ = var('x, y, z')  # optional - internet
         sage: f = sin(x^2) + y^z   # optional - internet
         sage: f.integrate(x, algorithm="mathematica_free")   # optional - internet
-        x*y^z + sqrt(1/2)*sqrt(pi)*fresnels(sqrt(2)*x/sqrt(pi))
+        x*y^z + sqrt(1/2)*sqrt(pi)*fresnel_sin(sqrt(2)*x/sqrt(pi))
 
     We can also use Sympy::
 

From 2eabdcbcba591b9eaabfc35c45e9cd9d35c139b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Labb=C3=A9?= <slabqc@gmail.com>
Date: Thu, 20 Dec 2018 11:11:15 +0100
Subject: [PATCH 2/4] 25501: Python 3 care

---
 src/sage/symbolic/integration/external.py | 46 +++++++++++------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/sage/symbolic/integration/external.py b/src/sage/symbolic/integration/external.py
index 6b9fdc046c2..b0f20373812 100644
--- a/src/sage/symbolic/integration/external.py
+++ b/src/sage/symbolic/integration/external.py
@@ -119,28 +119,28 @@ def request_wolfram_alpha(input, verbose=False):
 
         sage: from sage.symbolic.integration.external import request_wolfram_alpha
         sage: page_data = request_wolfram_alpha('integrate Sin[x]') # optional internet
-        sage: page_data.keys()                                      # optional internet
-        [u'queryresult']
-        sage: page_data['queryresult'].keys()                       # optional internet
-        [u'sponsorCategories',
-         u'parsetimedout',
-         u'success',
-         u'timedoutpods',
-         u'parsetiming',
-         u'id',
-         u'timedout',
-         u'related',
-         u'numpods',
-         u'host',
-         u'version',
-         u'encryptedParsedExpression',
-         u'encryptedEvaluatedExpression',
-         u'error',
-         u'timing',
-         u'datatypes',
-         u'server',
-         u'pods',
-         u'recalculate']
+        sage: sorted(page_data.keys())                              # optional internet
+        ['queryresult']
+        sage: sorted(page_data['queryresult'].keys())               # optional internet
+        ['datatypes',
+         'encryptedEvaluatedExpression',
+         'encryptedParsedExpression',
+         'error',
+         'host',
+         'id',
+         'numpods',
+         'parsetimedout',
+         'parsetiming',
+         'pods',
+         'recalculate',
+         'related',
+         'server',
+         'sponsorCategories',
+         'success',
+         'timedout',
+         'timedoutpods',
+         'timing',
+         'version']
 
     """
     # import compatible with py2 and py3
@@ -302,7 +302,7 @@ def symbolic_expression_from_mathematica_string(mexpr):
     expr = expr.replace('[', '(').replace(']', ')')
     expr = expr.replace('{', '[').replace('}', ']')
     lsymbols = symbol_table['mathematica'].copy()
-    autotrans = [unicode.lower,      # Try it in lower case
+    autotrans = [lambda x:x.lower(),      # Try it in lower case
                 un_camel,      # Convert `CamelCase` to `camel_case`
                 lambda x: x     # Try the original name
                 ]

From 76ad1cb684c219ce283cbbebe7281655fd18bff7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Labb=C3=A9?= <slabqc@gmail.com>
Date: Thu, 20 Dec 2018 11:30:49 +0100
Subject: [PATCH 3/4] 25501: fix back for Python 2

---
 src/sage/symbolic/integration/external.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/sage/symbolic/integration/external.py b/src/sage/symbolic/integration/external.py
index b0f20373812..5ed6e2135ea 100644
--- a/src/sage/symbolic/integration/external.py
+++ b/src/sage/symbolic/integration/external.py
@@ -118,10 +118,10 @@ def request_wolfram_alpha(input, verbose=False):
     EXAMPLES::
 
         sage: from sage.symbolic.integration.external import request_wolfram_alpha
-        sage: page_data = request_wolfram_alpha('integrate Sin[x]') # optional internet
-        sage: sorted(page_data.keys())                              # optional internet
+        sage: page_data = request_wolfram_alpha('integrate Sin[x]')      # optional internet
+        sage: [str(a) for a in sorted(page_data.keys())]                 # optional internet
         ['queryresult']
-        sage: sorted(page_data['queryresult'].keys())               # optional internet
+        sage: [str(a) for a in sorted(page_data['queryresult'].keys())]  # optional internet
         ['datatypes',
          'encryptedEvaluatedExpression',
          'encryptedParsedExpression',

From f083393998f39847e4cf93a65ee72036c74506f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Labb=C3=A9?= <slabqc@gmail.com>
Date: Mon, 31 Dec 2018 13:32:50 +0100
Subject: [PATCH 4/4] 25501: adding missing optional tags

---
 src/sage/symbolic/integration/external.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/sage/symbolic/integration/external.py b/src/sage/symbolic/integration/external.py
index 5ed6e2135ea..e8fc81f4662 100644
--- a/src/sage/symbolic/integration/external.py
+++ b/src/sage/symbolic/integration/external.py
@@ -237,8 +237,8 @@ def parse_moutput_from_json(page_data, verbose=False):
 
     TESTS::
 
-        sage: page_data = request_wolfram_alpha('Integrate(Sin[z], y)')
-        sage: parse_moutput_from_json(page_data)
+        sage: page_data = request_wolfram_alpha('Integrate(Sin[z], y)')  # optional internet
+        sage: parse_moutput_from_json(page_data)                         # optional internet
         Traceback (most recent call last):
         ...
         ValueError: asking wolframalpha.com was not successful