<pre> 
#!/usr/bin/env python3
#=====================================================================================
# Retrieve a URL from the web 
# Set request headers
# Get response headers
#=====================================================================================
import cgi
import urllib.request, urllib.error
import cgitb
cgitb.enable()

print("Content-Type: text/html \n")           #required http response header (w/ extra line)
print('&lt;br>')

#----------------------------------------------------------------
# Obtain (and set up) form entry fields
#----------------------------------------------------------------
elements = cgi.FieldStorage()                 #obtain the http for elements

url     = elements.getvalue('url')     or ''  #entry fields             
method  = elements.getvalue('method')  or ''               
headers = elements.getvalue('headers') or ''               
body    = elements.getvalue('body')    or ''               
format  = elements.getvalue('format')  or 'HTML'               

if not method: method = 'GET'
GET_checked  = 'checked' if method == 'GET'  else '' 
POST_checked = 'checked' if method == 'POST' else '' 
PUT_checked  = 'checked' if method == 'PUT'  else '' 
HEAD_checked = 'checked' if method == 'HEAD' else '' 

if not elements:
    headers = 'Referer: https://samsultan.com/   \n'                     \
            + 'Authorization: Basic student:123 \n'                     \
            + 'user-agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) \n' \
            + 'cookie: greeting=Hello; from=World \n'                   \
            + 'custom-name: Sam Sultan \n'                              \
            + 'custom-id: 123456'

msg = ''                                      #global variable

##################################################################################
# Function to validate entry fields
##################################################################################
def validate():
    global msg                                #needed to modify global variable
                                              #no need if access to global is read only
    if not url.startswith('http'):
        msg = '*** URL must start with http(s)://'
    if url=='':
        msg = '*** Please enter URL'

##################################################################################
# display: Display the form 
##################################################################################
def display():

    print(F'''
        &lt;!DOCTYPE html>
        &lt;html>
        &lt;head>
        &lt;title>Get URL with Headers and Body&lt;/title>
        &lt;style type="text/css">
            #Layer0 {{position:absolute; left:0px;   top:0px;   width:100%; height:60px; background-color:#cccccc  }}
            #Layer1 {{position:absolute; left:30%;   top:0px;   width:100%; color:#222222  }}
            #Layer2 {{position:absolute; left:20px;  top:80px;  width:780px; }}
            #Layer3 {{position:absolute; left:20px;  top:380px; width:100%;  }}
            #Layer4 {{position:absolute; left:20px;  top:480px; width:100%;  }}
            span    {{font-weight:bold; cursor:pointer}}
        &lt;/style>
        &lt;/head>
        &lt;body bgcolor='lightyellow'>
        &lt;div id='Layer0'>&nbsp;
        &lt;/div>
        &lt;div id='Layer1'>
        &lt;h2>HTTP*Tester &lt;sup>&copy;&lt;/sup> - &lt;font size=3>(Developed by: Sam Sultan)&lt;/h2>
        &lt;/div>

        &lt;div id='Layer2'>
        &lt;form METHOD=POST >                
        &lt;fieldset style='width:750'>
        &lt;legend>Get a URL with Request Headers and Body&lt;/legend> 

        &lt;table border='0'>
    ''')
    print("&lt;tr>&lt;td> Location &lt;/td>")
    print("&lt;td>&lt;input type=text id=url name=url style='width:650' placeholder=https:// value='" + url + "'>") 
    print("&lt;tr>&lt;td>&lt;td> GET")
    print("&lt;input type=radio name=method value=GET",  GET_checked, ">")
    print(" POST")
    print("&lt;input type=radio name=method value=POST", POST_checked, ">")
    print(" PUT")
    print("&lt;input type=radio name=method value=PUT",  PUT_checked, ">")
    print(" HEAD")
    print("&lt;input type=radio name=method value=HEAD", HEAD_checked, ">")
    print("&lt;tr>&lt;td valign=top> Headers &lt;br>")
    print("&lt;td>&lt;textarea name=headers style='width:650;height:110'>" + str(headers) + "&lt;/textarea>") 
    print("&lt;tr>&lt;td valign=top> Body &lt;br>")
    print("&lt;td>&lt;textarea name=body style='width:650;height:50'>" + str(body) + "&lt;/textarea>") 

    print(''' 
        &lt;/tr>
        &lt;tr>
        &lt;td>&lt;/td>
        &lt;td>&lt;input type='submit' value='  Retrieve  '>
            &nbsp;&nbsp;&nbsp;
    ''')

    html_checked = ''
    text_checked = ''    
    if format == 'HTML': html_checked = 'checked'
    else:                text_checked = 'checked'
    print('as HTML &lt;input type=radio name=format value=HTML', html_checked, '>')
    print('as TEXT &lt;input type=radio name=format value=TEXT', text_checked, '>')

    print("&lt;font color='red'>" + msg + "&lt;/font>&lt;/td>")
    print('''
        &lt;/tr>
        &lt;/table>       
        &lt;/fieldset>
        &lt;/form>
        &lt;/div>
        
        &lt;div id='Layer3'>
        &lt;i>Example:&lt;/i> 
        &lt;span onclick=document.getElementById('url').value=this.innerHTML>https://google.com&lt;/span>
        &lt;br>
        &lt;i>Example:&lt;/i> 
        &lt;span onclick=document.getElementById('url').value=this.innerHTML>http://workshop.sps.nyu.edu/~sultans/python/demo/7adv/response.py?env=no&lt;/span>
        &lt;br>
        &lt;i>Example:&lt;/i> 
        &lt;span onclick=document.getElementById('url').value=this.innerHTML>http://workshop.sps.nyu.edu/~sultans/python/demo/3webDB/form2.py?name1=value1&name2=value2&lt;/span>
        &lt;br>
        &lt;i>Example:&lt;/i> 
        &lt;span onclick=document.getElementById('url').value=this.innerHTML>http://workshop.sps.nyu.edu/~sultans/util/rest/REST.py?user=demo&pswd=demo&db=demo&sql=show tables &lt;/span>
        &lt;hr>
        &lt;/div>
    ''')
    
##################################################################################
#encode64: encode user:pswd to base64
##################################################################################
def encode64(input):
    import base64                                   #encode base64

    input = input.replace('Basic','')               #remove 'Basic'
    input = input.strip()                           #trim whitespace
    bArray      = input.encode()                    #convert the string to byte array
    encoded_arr = base64.b64encode(bArray)          #encode it base64
    encoded_str = encoded_arr.decode()              #convert the byte array back to string
    encoded_str = 'Basic ' + encoded_str            #re-add 'Basic'
    return(encoded_str)

##################################################################################
#do_request: Build a request object, including url, headers and body
#            Send the request
##################################################################################
def do_request():
    global request,response
        
    #--------------------------------------------------------
    # Encode the url, space --> %20
    #--------------------------------------------------------
#   url2 = urllib.request.pathname2url(url)                         #urlencode it                 
    url2 = url.replace(' ','%20')                                   #replace spaces with %20  

    #--------------------------------------------------------
    # if GET,          parameters at the end of the URL
    # if POST,PUT,HEAD parameters must be in body
    #--------------------------------------------------------    
    try:
        (url_only,param) = url2.split('?')                          #split the url on ?
    except:                                                         #if no ?
        url_only = url2                                             #url_only is simply url
        param    = ''                                               #there are no parameters
    
    if method != 'GET':                                             #if method POST, PUT, HEAD
        url2 = url_only                                             #should only take url (ie. no paramters)

    payload = body.encode()                                         #encode the body to byte array
             
    #--------------------------------------------------------
    # Build the request headers
    #--------------------------------------------------------
    headerDict   = {}                                               #create empty dictionary
    header_lines = headers.split('\n')                              #split the incoming headers on \n
    for line in header_lines:                                       #loop thru all the entered headers
        if line == '': continue                                     #if the header line is empty, skip it
        line = line.strip()                                         #get rid of begin/end whitespaces
        (name,value) = line.split(':',1)                            #split on first :
        if name == 'Authorization':
            value = encode64(value)                                 #Basic c3R1ZGVudDoxMjM= 
        headerDict[name] = value
        
    #----------------------------------------------------------
    # Set up the request object with url,method,headers,body
    #----------------------------------------------------------
    request = urllib.request.Request(url2, method=method, headers=headerDict, data=payload)    #set up the request object

    #----------------------------------------------------------
    # Obtain info from request object and print
    #----------------------------------------------------------
    reqURL     = request.get_full_url()                             #get the full url and param
    reqMethod  = request.get_method()                               #get the request method
    reqHeaders = request.header_items()                             #get the list of request headers 

    print('&lt;table border=1>')
    print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>Method: &lt;td>', reqMethod)
    print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>URL:    &lt;td>', reqURL)
    if method == 'GET':
        print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>Params: &lt;td>', param)
    else:
        print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>Request body: &lt;td>', str(payload) )
        
#   print(reqHeaders,'&lt;br>')
    for (header,value) in reqHeaders:                               #print each header name/value individually
        print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>', header +": &lt;td> "+ value)
    print('&lt;/table>')

    #--------------------------------------------------------
    # Make the request
    # Receive a response object
    #--------------------------------------------------------
    print('&lt;br>')
    print('&lt;b>&lt;i>&lt;font color=blue>Requesting the url ... &lt;/font>&lt;/b>&lt;/i>&lt;br>')

    try:
        response = urllib.request.urlopen(request)                  #retrieve URL, returns a HTTPResponse object
    except (urllib.error.HTTPError, urllib.error.URLError) as e:
        print('URL could not be reached -', e)
        exit()


##################################################################################
#do_response: Retrieve the response, including headers
#             Print the response
##################################################################################
def do_response():
    global request,response

    respURL      = response.geturl()                        #get the returned url
    respStatus   = response.getcode()                       #get the returned status code
    respHeaders  = response.info()                          #get the returned headers

    print('&lt;table border=1>')
    print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>Status: &lt;td>', respStatus)
    print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>URL:    &lt;td>', respURL)
#   print(respHeaders,'&lt;br>')
    for (header,value) in respHeaders.items():              #print each header name/value individually
        print('&lt;tr>&lt;td bgcolor=lightgray>&lt;b>'+ header +": &lt;td>"+ value)
    print('&lt;/table>')

    print('&lt;br>&lt;hr>')
    print('&lt;h2 style="color:red">&lt;i>The Server Response...&lt;/i>&lt;/h2>')

    content = response.read()                               #read it into a bytes object 

    content = content.decode('ascii', 'ignore')             #decode from byte array to string
                                                            #ignore if char is not ascii
#   print('&lt;br>')
    if format == 'TEXT':
        content = content.replace('&lt;','&lt;')               #replace all opening tags &lt; with &lt;
        content = content.replace('\n','&lt;br>')              #replace all \n to &lt;br>
    print(content)                                          #print the returned content
    print('&lt;br>')

#######################################################################################
# main code
#######################################################################################
if elements:                                    #if data was entered in the form 
    validate()                                  #validate entered data                            

display()                                       #display the form

if elements and not msg:                        #if data was entered in the form, and no errors 
    print('&lt;div id=layer4>')
    print("&lt;b style='color:blue'>---The REQUEST PACKET-----------------------------------------------&lt;/b>&lt;br>")
    do_request()

    print('&lt;br>')
    print("&lt;b style='color:red'>---The RESPONSE PACKET----------------------------------------------&lt;/b>&lt;br>")
    do_response()

    print('&lt;/div>')


#=== link to see the python code ================================================
import os
import sys
sys.path.insert(0,'/home/s/sultans/web/python/demo')
import zCode                          #import func to display the Python code
filename = os.path.abspath(__file__)  #get absolute file name 
zCode.display(filename)               #call it
#================================================================================

</pre>