Skip to content

Commit e3691e8

Browse files
author
shibing624
committed
deal with write and read file thread.
1 parent 101bf1a commit e3691e8

8 files changed

Lines changed: 228 additions & 44 deletions

File tree

‎23thread/file_thread_demo.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import fcntl
2+
import threading
3+
import time
4+
5+
6+
def writetoTxt(txtFile):
7+
id = threading.currentThread().getName()
8+
with open(txtFile, 'a') as f:
9+
fcntl.flock(f.fileno(), fcntl.LOCK_EX) # 加锁
10+
print("{0} acquire lock".format(id))
11+
f.write("write from {0} \r\n".format(id))
12+
fcntl.flock(f.fileno(), fcntl.LOCK_UN) # release
13+
time.sleep(3)
14+
15+
16+
# 在with块外，文件关闭，自动解锁
17+
print("{0} exit".format(id))
18+
for i in range(5):
19+
myThread = threading.Thread(target=writetoTxt, args=("test.txt",))
20+
myThread.start()

‎23thread/file_thread_demo2.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import threading
2+
import time
3+
import json
4+
5+
6+
def writetoTxt(txtFile, mutex, json_file):
7+
id = threading.currentThread().getName()
8+
mutex.acquire(10)
9+
print("Thread {0} acquire lock".format(id))
10+
data = load_json(json_file)
11+
print('data', data, type(data))
12+
time.sleep(1.5)
13+
print('start add new data')
14+
15+
data.update({id: {"username": str(id), "sex": 'man'}})
16+
save_json(data, json_file)
17+
print('new data', data, type(data))
18+
time.sleep(1.5)
19+
mutex.release()
20+
print("Thread {0} exit".format(id))
21+
22+
23+
def load_json(json_file):
24+
with open(json_file, 'r') as f:
25+
data = json.load(f)
26+
return data
27+
28+
29+
def save_json(data, json_file):
30+
with open(json_file, 'w') as f:
31+
json.dump(data, f, indent=4)
32+
33+
34+
if __name__ == '__main__':
35+
a = {
36+
"lili": {"username": "lili_01", "sex": "woman", "age": 12, "address": "beijing,china."},
37+
"lucy": {"username": "lucy_1", "sex": "woman", "age": 32}
38+
}
39+
json_file = "a.json"
40+
save_json(a, json_file)
41+
42+
mutex = threading.Lock()
43+
for i in range(5):
44+
myThread = threading.Thread(target=writetoTxt, args=("test.txt", mutex, json_file))
45+
myThread.start()

‎23thread/log.py‎

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎23thread/logger.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@author:XuMing（[email protected])
4+
@description:
5+
"""
6+
import logging
7+
8+
9+
def get_logger(name, log_file=None, log_level='DEBUG'):
10+
"""
11+
logger
12+
:param name: 模块名称
13+
:param log_file: 日志文件，如无则输出到标准输出
14+
:param log_level: 日志级别
15+
:return:
16+
"""
17+
logger = logging.getLogger(name)
18+
logger.setLevel(log_level.upper())
19+
formatter = logging.Formatter(
20+
'[%(levelname)7s %(asctime)s %(processName)s %(threadName)s %(module)s:%(lineno)d] %(message)s',
21+
datefmt='%Y%m%d %I:%M:%S')
22+
if log_file:
23+
f_handle = logging.FileHandler(log_file)
24+
f_handle.setFormatter(formatter)
25+
logger.addHandler(f_handle)
26+
handle = logging.StreamHandler()
27+
handle.setFormatter(formatter)
28+
logger.addHandler(handle)
29+
return logger
30+
31+
32+
logger = get_logger(__name__, log_file=None, log_level='DEBUG')
33+
34+
35+
def set_log_level(log_level='INFO'):
36+
logger.setLevel(log_level.upper())
37+
38+
39+
if __name__ == '__main__':
40+
logger.debug('hi')
41+
logger.info('hi')
42+
logger.error('hi')
43+
logger.warning('hi')
44+
set_log_level('info')
45+
logger.debug('hi') # ignore
46+
logger.info('hi')
47+
logger.error('hi')
48+
logger.warning('hi')
49+
import threading
50+
import time
51+
52+
53+
def doubler(number, logger):
54+
logger.info("aaaa")
55+
logger.warning("bbbb")
56+
logger.error("cccc")
57+
58+
result = number * 2
59+
time.sleep(5)
60+
logger.info('result: {}'.format(result))
61+
62+
63+
thread_names = ['Mike', 'George', 'Wanda']
64+
# logger = get_logger('test_thread')
65+
for i in range(3):
66+
my_thread = threading.Thread(
67+
target=doubler, name=thread_names[i], args=(i, logger))
68+
my_thread.start()

‎23thread/print_time.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,14 @@
77
import time
88
import threading
99

10+
1011
def send_online_notification(user):
1112
print(user)
1213
while True:
1314
print('I\'m Still Alive!! ' + time.strftime('%y/%m/%d-%H:%M:%S', time.localtime()))
1415
time.sleep(5)
1516

17+
1618
username = 'x'
1719
t = threading.Thread(target=send_online_notification, args=(username,))
1820
t.setDaemon(True)
@@ -22,4 +24,4 @@ def send_online_notification(user):
2224
print('bot embed.')
2325
while True:
2426
print("a")
25-
time.sleep(2)
27+
time.sleep(2)

‎23thread/spider_thread.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import requests
2+
from multiprocessing import Pool # 进程池
3+
from multiprocessing.dummy import Pool as ThreadPool # 线程池
4+
from logger import get_logger
5+
logger = get_logger(__name__, 'log.txt')
6+
7+
def get_data_from_url(url):
8+
txt = requests.get(url).text
9+
logger.info('url:{}, size:{}'.format(url, len(txt)))
10+
return txt
11+
12+
13+
if __name__ == '__main__':
14+
url_list = ['https://www.jianshu.com/p/f8c5719e5af4',
15+
'https://github.com/zhangjunhd/reading-notes/blob/master/literature/why-do-people-live.md',
16+
'https://github.com/hli1221']
17+
18+
tpool = ThreadPool(20) # 创建一个线程池，20个线程数
19+
data_list = tpool.map(get_data_from_url, url_list) # 将任务交给线程池，所有url都完成后再继续执行，与python的map方法类似
20+
21+
tpool.close()
22+
tpool.join()
23+
head_data = [i[:100] for i in data_list if i]
24+
25+
print(len(data_list))
26+
print(head_data)
27+
28+
print("*" * 43)
29+
pool = Pool(4)
30+
data_list = pool.map(get_data_from_url, url_list) # 与线程池的map方法工作原理一致
31+
32+
pool.close()
33+
pool.join()
34+
head_data = [i[:100] for i in data_list if i]
35+
36+
print(len(data_list))
37+
print(head_data)

‎23thread/timer_demo.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,30 @@
77
import threading
88
import time
99

10+
1011
def hello(name):
1112
print("hello %s\n" % name)
1213

1314
global timer
1415
timer = threading.Timer(2.0, hello, ["Hawk"])
1516
timer.start()
1617

18+
1719
def is_end(limit_time):
1820
timer = threading.Timer(limit_time, is_end, [limit_time])
1921
timer.start()
2022
return True
2123

24+
2225
def end():
2326
return True
2427

28+
2529
def timer_end():
26-
timer = threading.Timer(3,end)
30+
timer = threading.Timer(3, end)
2731
timer.start()
32+
33+
2834
if __name__ == "__main__":
2935
# hello('girl')
30-
print(is_end(2))
36+
print(is_end(2))

‎tool/logger.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
@author:XuMing（[email protected])
4+
@description:
5+
"""
6+
import logging
7+
8+
9+
def get_logger(name, log_file=None, log_level='DEBUG'):
10+
"""
11+
logger
12+
:param name: 模块名称
13+
:param log_file: 日志文件，如无则输出到标准输出
14+
:param log_level: 日志级别
15+
:return:
16+
"""
17+
logger = logging.getLogger(name)
18+
logger.setLevel(log_level.upper())
19+
formatter = logging.Formatter('[%(levelname)7s %(asctime)s %(module)s:%(lineno)d] %(message)s',
20+
datefmt='%Y%m%d %I:%M:%S')
21+
if log_file:
22+
f_handle = logging.FileHandler(log_file)
23+
f_handle.setFormatter(formatter)
24+
logger.addHandler(f_handle)
25+
handle = logging.StreamHandler()
26+
handle.setFormatter(formatter)
27+
logger.addHandler(handle)
28+
return logger
29+
30+
31+
logger = get_logger(__name__, log_file=None, log_level='DEBUG')
32+
33+
34+
def set_log_level(log_level='INFO'):
35+
logger.setLevel(log_level.upper())
36+
37+
38+
if __name__ == '__main__':
39+
logger.debug('hi')
40+
logger.info('hi')
41+
logger.error('hi')
42+
logger.warning('hi')
43+
set_log_level('info')
44+
logger.debug('hi') # ignore
45+
logger.info('hi')
46+
logger.error('hi')
47+
logger.warning('hi')

0 commit comments

Comments
 (0)