Full Code of Max00355/HTTPLang for AI

Repository: Max00355/HTTPLang
Branch: master
Commit: 768997805692
Files: 20
Total size: 13.3 KB

Directory structure:
gitextract_r8qt4w87/

├── .gitignore
├── CHANGELOG.md
├── LICENSE.md
├── README.md
├── examples/
│   ├── everything.http
│   ├── get.http
│   ├── loop.http
│   └── test_cookie_and_post.http
├── grammar.txt
├── httplang/
│   ├── __init__.py
│   ├── evaluate.py
│   ├── global_data.py
│   ├── make_request.py
│   ├── parse.py
│   ├── test.httpl
│   └── tokenize.py
├── httplang.py
├── requirements.txt
├── setup.py
└── test.httpl

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
*.pyc
*.swp
*.swo

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]

# C extensions
*.so

# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover

# Translations
*.mo
*.pot

# Django stuff:
*.log

# Sphinx documentation
docs/_build/

# PyBuilder
target/

# OS X
*.DS*

# Test images
*.jpg

================================================
FILE: CHANGELOG.md
================================================
Change Log
==========

v0.1.0
* Introduced loops
* First official version


================================================
FILE: LICENSE.md
================================================
The MIT License (MIT)

Copyright (c) 2015 Frankie Primerano

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.


================================================
FILE: README.md
================================================
About
=====

HTTPLang is a scripting language that makes writing HTTP request routines simpler.

Current Version
===============

0.2.0

What could I use it for?
====

- Testing APIs
- Web Scraping
-


Installation
====
* Clone the repo
* Run `python setup.py install`

Usage
===
`httplang l<file.http>`

Documentation
=============

Important Note
--------------

HTTPLang is very strict about things like spaces. Make sure that your code matches the spacing and what not exactly as shown in the examples. If not there will be errors.

Examples
--------


```

set URL "http://google.com"
do GET "/
show RESPONSE

``` 

```
set URL "http://google.com"
label "loop"
do GET /
show STATUS
goto "label"

```

```

set URL "http://somesite.com"
set POSTDATA "username=myUsername&password=myPassword is this"
do POST "/login"
set COOKIE TMPCOOKIE
do GET "/usercp"

```



================================================
FILE: examples/everything.http
================================================
set URL http://google.com/
do GET /
set URL $URL
set POSTDATA a=1,b=2,c=3,d=4
set COOKIE $TMPCOOKIE
do POST /


================================================
FILE: examples/get.http
================================================
set URL http://google.com/
do GET /
show $HTML


================================================
FILE: examples/loop.http
================================================
set URL http://google.com
loop 2
do GET /
show $STATUS
endloop


================================================
FILE: examples/test_cookie_and_post.http
================================================
set URL http://localhost:5000/
do GET /
set POSTDATA one=1,two=2
do POST /
set COOKIE $TMPCOOKIE
do GET /


================================================
FILE: grammar.txt
================================================
<set> := <global_var> <string> 
<do> := <method> <string>
<method> := GET | POST | PUT | DELETE | PATCH
<global_var> := URL | TMPCOOKIE | COOKIE | HTML | POSTDATA | USERAGENT | STATUS | VALUE | LINKS
<show> := <global_var>
<condition> := <expr> <goto>
<expr> := (<global_var> | <string> | <int>) <operator> (<global_var> | <string> | <int> ) 
<label> := <string>
<goto> := <int> | <label>
<operator> := "==" | "!=" | ">=" | "<="
<string> := "\"(.*)?\""
<int> := "[0-9]+"


================================================
FILE: httplang/__init__.py
================================================
import evaluate
import global_data
import make_request
import parse
import tokenize


================================================
FILE: httplang/evaluate.py
================================================
import operator
import parse 
import tokenize
import global_data
import make_request

operators = {
    "==":operator.eq,
    "!=":operator.ne,
    ">=":operator.ge,
    "<=":operator.le,
    ">":operator.gt,
    "<":operator.lt
}


def evaluate(ast):
    ast = list(ast)
    switch = {
        "DO":do,
        "SET":set_,
        "SHOW":show,
        "GOTO":goto,
        "CONDITION":condition
    }
    while global_data.line_on < len(ast):
        on = ast[global_data.line_on]
        tt = on.token_type
        if switch.get(tt):
            switch[tt](on)
        global_data.line_on += 1

def do(line):
    method = line.left.lexeme
    path = line.right.lexeme
    switch = {
        "GET":make_request.GET,
        "POST":make_request.POST,
        "PUT":make_request.PUT,
        "DELETE":make_request.DELETE,
        "PATCH":make_request.PATCH
    }
    switch[method](path)

def set_(line):
    global_var = line.left.lexeme
    val = line.right.token_type
    if val == "GLOBAL":
        val = global_data.GLOBALS[line.right.lexeme]
    elif val == "STRING":
        val = line.right.lexeme
    global_data.GLOBALS[global_var] = val

def show(line):
    if line.left.token_type == "GLOBAL":
        print(global_data.GLOBALS[line.left.lexeme])
    elif line.left.token_type == "STRING":
        print(line.left.lexeme)

def goto(line):
    global_data.line_on = global_data.labels[line.left.lexeme]

def condition(line):
    condition = line.left.lexeme
    larg = line.left.left
    rarg = line.left.right
    if larg.token_type == "GLOBAL":
        larg = global_data.GLOBALS[larg.lexeme]
    else:
        larg = larg.lexeme
    if rarg.token_type == "GLOBAL":
        rarg = global_data.GLOBALS[rarg.lexeme]
    else:
        rarg = rarg.lexeme
    if operators[condition](str(larg), str(rarg)): # THis is a temp fix, when integers are more important this can't be used.
        goto(line.right)

if __name__ == "__main__":
    f = "test.httpl"
    tokens = tokenize.getTokens(open(f))
    evaluate(parse.program(tokens))



================================================
FILE: httplang/global_data.py
================================================
labels = {}

line_on = 0

GLOBALS = {
    "URL":"",
    "SETCOOKIE":"",
    "COOKIE":"",
    "HTML":"",
    "POSTDATA":"",
    "USERAGENT":"",
    "STATUS":"",
    "LINKS":"",
}



================================================
FILE: httplang/make_request.py
================================================
try:
    import urllib2 as urllib
except:
    import urllib.request as urllib
import global_data
import sys
from global_data import GLOBALS

def setValues(response):
    GLOBALS['RESPONSE'] = response.read()
    GLOBALS['STATUS'] = response.code
    GLOBALS['SETCOOKIE'] = response.info()['set-cookie']

def getURL():
    url = GLOBALS['URL']
    if not url:
        sys.exit("Variable Error: URL is not set line {}".format(global_data.line_on))
    return url   

def getOpener():
    opener = urllib.build_opener()
    opener.addheaders.append(('Cookie', GLOBALS['COOKIE']))
    opener.addheaders.append(("User-Agent", GLOBALS['USERAGENT']))
    return opener

def GET(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url)
    response = opener.open(request)
    setValues(response)

def POST(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url, GLOBALS['POSTDATA'])
    response = opener.open(request)
    setValues(response)

def PATCH(path):
    pass

def DELETE(path):
    pass

def PUT(path):
    pass


================================================
FILE: httplang/parse.py
================================================
import tokenize
import sys
import global_data
import itertools

global line
line = 0

class AST: 
    def __init__(self, left, right, tt, lexeme, l):
        self.left = left
        self.right = right
        self.token_type = tt
        self.lexeme = lexeme
        self.line = l

def program(tokens):
    global line
    switch = {
        "DO":do,
        "SET":set_,
        "SHOW":show,
        "LABEL":label,
        "GOTO":goto,
        "CONDITION":condition
    }
    for token in tokens:
        tt = token['tokenType']
        if switch.get(tt):
            line += 1
            yield switch[tt](tokens) 
        else:
            sys.exit("Parse Error: Don't know what to do with {} line {}".format(token, line))

def condition(tokens):
    condition_expr = expr(tokens)
    goto_check = tokens.next()
    if goto_check['tokenType'] != "GOTO":
        sys.exit("Parse Error: goto expected after condition line {}".format(line))
    goto_part = goto(tokens)
    return AST(condition_expr, goto_part, "CONDITION", None, line)

def do(tokens):
    method_val = method(tokens)
    location = string(tokens)
    return AST(method_val, location, "DO", None, line)

def set_(tokens):
    global_val = global_var(tokens)
    tokens, copy = itertools.tee(tokens)
    type_check = copy.next()['tokenType']
    if type_check == "GLOBAL":
        location = global_var(tokens)
    else:
        location = string(tokens)
    return AST(global_val, location, "SET", None, line)

def expr(tokens):
    left_arg = tokens.next()
    if left_arg['tokenType'] not in ["GLOBAL", "STRING", "INTEGER"]:
        sys.exit("Parse Error: Invalid left argument {} line {}".format(left_arg['lexeme'], line))

    op = tokens.next()
    if op['tokenType'] != "OPERATOR":
        sys.exit("Parse Error: Invalid operator {} line {}".format(op['lexeme'], line))

    right_arg = tokens.next()
    if right_arg['tokenType'] not in ["GLOBAL", "STRING", "INTEGER"]:
        sys.exit("Parse Error: Invalid right argument {} line {}".format(right_arg["lexeme"], line))
    return AST(
            AST(None, None, left_arg['tokenType'], left_arg['lexeme'], line),
            AST(None, None, right_arg["tokenType"], right_arg['lexeme'], line),
            "OPERATOR",
            op['lexeme'],
            line)

def show(tokens):
    tokens, copy = itertools.tee(tokens)
    type_check = copy.next()['tokenType']
    if type_check == "STRING":
        variable_name = string(tokens)
    else:
        variable_name = global_var(tokens)
    return AST(variable_name, None, "SHOW", None, line)

def label(tokens):
    label_name = string(tokens)
    global_data.labels[label_name.lexeme] = line - 1
    return AST(label_name, None, "LABEL", None, line)

def goto(tokens):
    label_name = string(tokens)
    return AST(label_name, None, "GOTO", None, line)

def string(tokens):
    string_val = tokens.next()
    if string_val['tokenType'] != "STRING":
        sys.exit("Parse Error: {} is not a STRING line {}".format(string_val['lexeme'], line))
    return AST(None, None, string_val['tokenType'], string_val['lexeme'], line)

def integer(tokens):
    int_val = tokens.next()
    if int_val['tokenType'] != "INTEGER":
        sys.exit("TypeError: {} is not an INTEGER line {}".format(int_val['lexeme'], line))
    try:
        int_val['lexeme'] = int(int_val['lexeme'])
    except:
        sys.exit("Type Error: {} is not an INTEGER")
                
    return AST(None, None, int_val['tokenType'], int_val['lexeme'], line)

def method(tokens):
    method_val = tokens.next()
    if method_val['tokenType'] != "METHOD":
        sys.exit("Type Error: {} is not a METHOD line {}".format(method_val['lexeme'], line))
    return AST(None, None, method_val['tokenType'], method_val['lexeme'], line)

def global_var(tokens):
    variable_val = tokens.next()
    if variable_val['tokenType'] == "GLOBAL_VAR":
        sys.exit("Type Error: {} is not a GLOBAL_VAR line {}".format(variable_val['lexeme'], line))
    return AST(None, None, variable_val['tokenType'], variable_val['lexeme'], line)

if __name__ == "__main__":
    print(list(program(tokenize.getTokens(open("test.httpl")))))



================================================
FILE: httplang/test.httpl
================================================
set URL "http://google.com"
do POST "/"
show COOKIE
set COOKIE SETCOOKIE
show COOKIE


================================================
FILE: httplang/tokenize.py
================================================
import re
import sys

tokens = {
        "^do$":"DO",
        "^set$":"SET",
        "^GET$|^POST$|^PUT$|^DELETE$|^PATCH$":"METHOD",
        "^URL$|^SETCOOKIE$|^COOKIE$|^RESPONSE$|^POSTDATA$|^USERAGENT$|^STATUS$|^LINKS$":"GLOBAL",
        "^show$":"SHOW",
        "^if$":"CONDITION",
        "^label$":"LABEL",
        "^goto$":"GOTO",
        ">|<|==|!=|>=|<=":"OPERATOR",
        "\"(.*?)\"":"STRING",
        "[0-9]+":"INTEGER"
}

def getTokens(stream):
    token = ""
    line = 1
    for char in stream.read():
        if char == " " or char == "\n":
            for token_check in tokens:
                check = re.findall(token_check, token)
                if check:
                    token = ""
                    yield {
                            "lexeme":check[0],
                            "tokenType":tokens[token_check]
                          }
                    break
            else:
                sys.exit("Invalid Token: {} on line {}".format(token, line))
            if char == "\n":
                line += 1
        else:
            token += char

if __name__ == "__main__":
    print(list(getTokens(open("test.httpl"))))


================================================
FILE: httplang.py
================================================
from httplang import *
import sys
import os

if len(sys.argv) < 2:
    sys.exit("Usage: python httplang.py <file>.httpl")

if not os.path.exists(sys.argv[1]):
    sys.exit("No file names {}".format(sys.argv[1]))

evaluate.evaluate(parse.program(tokenize.getTokens(open(sys.argv[1]))))


================================================
FILE: requirements.txt
================================================
requests



================================================
FILE: setup.py
================================================
from setuptools import setup
import sys

setup(name='HTTPLang',
      version='2.0.0',
      author='Frankie Primerano',
      author_email='max00355@gmail.com',
      packages=['httplang'],
      entry_points={
          'console_scripts': ['httplang=httplang:console_main'],
      },
      url='https://github.com/Max00355/HTTPLang',
      description='A scripting language to do HTTP routines.',
      classifiers=[
          'Operating System :: POSIX',
          'Programming Language :: Python',
          'Programming Language :: Python :: 2',
          'Programming Language :: Python :: 2.7',
          'Programming Language :: Python :: 3',
          'Programming Language :: Python :: 3.2',
          'Programming Language :: Python :: 3.3',
          'License :: OSI Approved :: MIT License',
          'Topic :: Utilities'
      ],
      )


================================================
FILE: test.httpl
================================================
label "start"
set URL "http://google.com"
do GET "/"
show "LOL"
show STATUS
if STATUS == 200 goto "start"