Experimenting multiprocessing with multi Threads requesting to the url in python.
Finally i got all the url html responce correctly it is working fine as expected.With multiprocessing and multi Threads.
def calc_square(number):
time.sleep(3)
print('Square:' , number * number)
result = number * number
print(os.getpid())
print(result)
def calc_quad(number):
print(os.getpid())
print('Quad:' , number * number * number * number)
def test1():
print(os.getpid())
print('test1 :' ,)
def test2():
print(os.getpid())
print('test2 :' ,)
def test3():
print(os.getpid())
print('test3 :' ,)
if __name__ == '__main__':
number = 7
start = time.time()
result = None
# p1 = multiprocessing.Process(target=calc_square, args=(number,))
# p2 = multiprocessing.Process(target=calc_quad, args=(number,))
p3 = multiprocessing.Process(target=run_test1)
p4 = multiprocessing.Process(target=run_test)
p5 = multiprocessing.Process(target=test3)
# p1.start()
# p2.start()
p3.start()
p4.start()
p5.start()
# p1.join()
# p2.join()
p3.join()
p4.join()
p5.join()
end = time.time()
print(f'Time taken {end - start}')
test.py
import os
import time
from Utility.NetworkRequest import getHtml_Str_From_Url
from Utility.Utility import create_thread
link_list = ['https://tutorialedge.net/python/concurrency/python-threadpoolexecutor-
tutorial/'
, 'https://www.machinelearningplus.com/python/parallel-processing-python/']
def run_test():
# count = 0
thread = create_thread(run_other)
thread.start()
print(hash(thread))
def run_other():
count = 0
while len(link_list) > count:
print("from test")
print("process id:- " + str(os.getpid()))
print("links: - " + str(link_list[count]))
getHtml = getHtml_Str_From_Url(link_list[count])
print(getHtml)
count += 1
save.py
import os
import time
from Utility.NetworkRequest import getHtml_Str_From_Url
from Utility.Utility import create_thread
link_list = ['https://realpython.com/intro-to-python-
threading/','https://pymotw.com/3/concurrent.futures/']
def run_test1():
thread = create_thread(run_other)
thread.start()
print(hash(thread))
def run_other():
count = 0
while len(link_list) > count:
print("from save")
print("process id:- " + str(os.getpid()))
print("links: - " + str(link_list[count]))
getHtml = getHtml_Str_From_Url(link_list[count])
print(getHtml)
count += 1
Comments
Post a Comment