Skip to content

Commit dac45ae

Browse files
Boris-codeBoris-code
committed
优化命令行
1 parent 84e1cf3 commit dac45ae

12 files changed

Lines changed: 269 additions & 59 deletions

File tree

‎docs/usage/BatchSpider.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class BatchSpiderTest(feapder.BatchSpider):
4242
4343
if __name__ == "__main__":
4444
spider = BatchSpiderTest(
45-
redis_key="xxx:xxxx", # redis中存放任务等信息的根key
45+
redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置
4646
task_table="", # mysql中的任务表
4747
task_keys=["id", "xxx"], # 需要获取任务表里的字段名，可添加多个
4848
task_state="state", # mysql中任务状态字段
@@ -137,7 +137,7 @@ def start_requests(self, task):
137137
```
138138
def crawl_test(args):
139139
spider = test_spider.TestSpider(
140-
redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key
140+
redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置
141141
task_table="batch_spider_task", # mysql中的任务表
142142
task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个
143143
task_state="state", # mysql中任务状态字段
@@ -251,7 +251,7 @@ def failed_request(self, request, response):
251251
def test_debug():
252252
spider = test_spider.TestSpider.to_DebugBatchSpider(
253253
task_id=1,
254-
redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key
254+
redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置
255255
task_table="batch_spider_task", # mysql中的任务表
256256
task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个
257257
task_state="state", # mysql中任务状态字段
@@ -282,7 +282,7 @@ from feapder import ArgumentParser
282282
283283
def crawl_test(args):
284284
spider = test_spider.TestSpider(
285-
redis_key="feapder:test_batch_spider", # redis中存放任务等信息的根key
285+
redis_key="feapder:test_batch_spider", # 分布式爬虫调度信息存储位置
286286
task_table="batch_spider_task", # mysql中的任务表
287287
task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个
288288
task_state="state", # mysql中任务状态字段

‎feapder/commands/create/create_item.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,15 @@ def convert_table_name_to_hump(self, table_name):
6565

6666
return table_hump_format
6767

68-
def get_item_template(self):
69-
template_path = os.path.abspath(
70-
os.path.join(__file__, "../../../templates/item_template.tmpl")
71-
)
68+
def get_item_template(self, item_type):
69+
if item_type == "Item":
70+
template_path = os.path.abspath(
71+
os.path.join(__file__, "../../../templates/item_template.tmpl")
72+
)
73+
else:
74+
template_path = os.path.abspath(
75+
os.path.join(__file__, "../../../templates/update_item_template.tmpl")
76+
)
7277
with open(template_path, "r", encoding="utf-8") as file:
7378
item_template = file.read()
7479

@@ -148,7 +153,7 @@ def save_template_to_file(self, item_template, table_name):
148153
if os.path.basename(os.path.dirname(os.path.abspath(item_file))) == "items":
149154
self._create_init.create()
150155

151-
def create(self, tables_name, support_dict):
156+
def create(self, tables_name, item_type, support_dict):
152157
input_tables_name = tables_name
153158

154159
tables_name = self.select_tables_name(tables_name)
@@ -161,7 +166,7 @@ def create(self, tables_name, support_dict):
161166
table_name = table_name[0]
162167

163168
columns = self.select_columns(table_name)
164-
item_template = self.get_item_template()
169+
item_template = self.get_item_template(item_type)
165170
item_template = self.create_item(
166171
item_template, columns, table_name, support_dict
167172
)

‎feapder/commands/create/create_spider.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,16 @@ def cover_to_underline(self, key):
4949
return key
5050

5151
def get_spider_template(self, spider_type):
52-
if spider_type == 1:
52+
if spider_type == "AirSpider":
5353
template_path = "air_spider_template.tmpl"
54-
elif spider_type == 2:
54+
elif spider_type == "Spider":
5555
template_path = "spider_template.tmpl"
56-
elif spider_type == 3:
56+
elif spider_type == "TaskSpider":
57+
template_path = "task_spider_template.tmpl"
58+
elif spider_type == "BatchSpider":
5759
template_path = "batch_spider_template.tmpl"
5860
else:
59-
raise ValueError("spider type error, support 1 2 3")
61+
raise ValueError("spider type error, only support AirSpider、 Spider、TaskSpider、BatchSpider")
6062

6163
template_path = os.path.abspath(
6264
os.path.join(__file__, "../../../templates", template_path)
@@ -66,26 +68,24 @@ def get_spider_template(self, spider_type):
6668

6769
return spider_template
6870

69-
def create_spider(self, spider_template, spider_name):
71+
def create_spider(self, spider_template, spider_name, file_name):
7072
spider_template = spider_template.replace("${spider_name}", spider_name)
73+
spider_template = spider_template.replace("${file_name}", file_name)
7174
spider_template = deal_file_info(spider_template)
7275
return spider_template
7376

74-
def save_spider_to_file(self, spider, spider_name):
75-
spider_underline = self.cover_to_underline(spider_name)
76-
spider_file = spider_underline + ".py"
77-
78-
if os.path.exists(spider_file):
79-
confirm = input("%s 文件已存在 是否覆盖 (y/n). " % spider_file)
77+
def save_spider_to_file(self, spider, spider_name, file_name):
78+
if os.path.exists(file_name):
79+
confirm = input("%s 文件已存在 是否覆盖 (y/n). " % file_name)
8080
if confirm != "y":
8181
print("取消覆盖 退出")
8282
return
8383

84-
with open(spider_file, "w", encoding="utf-8") as file:
84+
with open(file_name, "w", encoding="utf-8") as file:
8585
file.write(spider)
8686
print("\n%s 生成成功" % spider_name)
8787

88-
if os.path.basename(os.path.dirname(os.path.abspath(spider_file))) == "spiders":
88+
if os.path.basename(os.path.dirname(os.path.abspath(file_name))) == "spiders":
8989
self._create_init.create()
9090

9191
def create(self, spider_name, spider_type):
@@ -94,8 +94,12 @@ def create(self, spider_name, spider_type):
9494
print("爬虫命名不符合规范，请用蛇形或驼峰命名方式")
9595
return
9696

97-
if spider_name.islower():
98-
spider_name = tools.key2hump(spider_name)
97+
underline_format = self.cover_to_underline(spider_name)
98+
spider_name = tools.key2hump(underline_format)
99+
file_name = underline_format + ".py"
100+
101+
print(spider_name, file_name)
102+
99103
spider_template = self.get_spider_template(spider_type)
100-
spider = self.create_spider(spider_template, spider_name)
101-
self.save_spider_to_file(spider, spider_name)
104+
spider = self.create_spider(spider_template, spider_name, file_name)
105+
self.save_spider_to_file(spider, spider_name, file_name)

‎feapder/commands/create_builder.py‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
"""
1010
import argparse
1111

12+
from terminal_layout import Fore
13+
from terminal_layout.extensions.choice import Choice, StringStyle
14+
1215
import feapder.setting as setting
1316
from feapder.commands.create import *
1417

@@ -22,21 +25,13 @@ def main():
2225
spider.add_argument(
2326
"-s",
2427
"--spider",
25-
nargs="+",
26-
help="创建爬虫\n"
27-
"如 feapder create -s <spider_name> <spider_type> "
28-
"spider_type=1 AirSpider; "
29-
"spider_type=2 Spider; "
30-
"spider_type=3 BatchSpider;",
28+
help="创建爬虫 如 feapder create -s <spider_name>",
3129
metavar="",
3230
)
3331
spider.add_argument(
3432
"-i",
3533
"--item",
36-
nargs="+",
37-
help="创建item 如 feapder create -i test 则生成test表对应的item。 "
38-
"支持like语法模糊匹配所要生产的表。 "
39-
"若想生成支持字典方式赋值的item，则create -item test 1",
34+
help="创建item 如 feapder create -i <table_name> 支持模糊匹配 如 feapder create -i %%table_name%%",
4035
metavar="",
4136
)
4237
spider.add_argument(
@@ -73,21 +68,35 @@ def main():
7368
setting.MYSQL_DB = args.db
7469

7570
if args.item:
76-
item_name, *support_dict = args.item
77-
support_dict = bool(support_dict)
78-
CreateItem().create(item_name, support_dict)
71+
c = Choice(
72+
"请选择Item类型 (press <esc> to exit) ",
73+
["Item", "Item 支持字典赋值", "UpdateItem", "UpdateItem 支持字典赋值"],
74+
icon_style=StringStyle(fore=Fore.green),
75+
selected_style=StringStyle(fore=Fore.green),
76+
)
77+
78+
choice = c.get_choice()
79+
if choice:
80+
index, value = choice
81+
item_name = args.item
82+
item_type = "Item" if index <= 1 else "UpdateItem"
83+
support_dict = index in (1, 3)
84+
85+
CreateItem().create(item_name, item_type, support_dict)
7986

8087
elif args.spider:
81-
spider_name, *spider_type = args.spider
82-
if not spider_type:
83-
spider_type = 1
84-
else:
85-
spider_type = spider_type[0]
86-
try:
87-
spider_type = int(spider_type)
88-
except:
89-
raise ValueError("spider_type error, support 1, 2, 3")
90-
CreateSpider().create(spider_name, spider_type)
88+
c = Choice(
89+
"请选择爬虫模板 (press <esc> to exit) ",
90+
["AirSpider", "Spider", "TaskSpider", "BatchSpider"],
91+
icon_style=StringStyle(fore=Fore.green),
92+
selected_style=StringStyle(fore=Fore.green),
93+
)
94+
95+
choice = c.get_choice()
96+
if choice:
97+
index, spider_type = choice
98+
spider_name = args.spider
99+
CreateSpider().create(spider_name, spider_type)
91100

92101
elif args.project:
93102
CreateProject().create(args.project)

‎feapder/requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ loguru>=0.5.3
1717
influxdb>=5.3.1
1818
pyperclip>=1.8.2
1919
webdriver-manager>=3.5.3
20+
terminal-layout>=2.1.2

‎feapder/templates/batch_spider_template.tmpl‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ Created on {DATE}
88
"""
99

1010
import feapder
11+
from feapder import ArgumentParser
1112

1213

1314
class ${spider_name}(feapder.BatchSpider):
@@ -18,9 +19,9 @@ class ${spider_name}(feapder.BatchSpider):
1819
REDISDB_DB=0,
1920
MYSQL_IP="localhost",
2021
MYSQL_PORT=3306,
21-
MYSQL_DB="feapder",
22-
MYSQL_USER_NAME="feapder",
23-
MYSQL_USER_PASS="feapder123",
22+
MYSQL_DB="",
23+
MYSQL_USER_NAME="",
24+
MYSQL_USER_PASS="",
2425
)
2526

2627
def start_requests(self, task):
@@ -36,7 +37,7 @@ class ${spider_name}(feapder.BatchSpider):
3637

3738
if __name__ == "__main__":
3839
spider = ${spider_name}(
39-
redis_key="xxx:xxxx", # redis中存放任务等信息的根key
40+
redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置
4041
task_table="", # mysql中的任务表
4142
task_keys=["id", "xxx"], # 需要获取任务表里的字段名，可添加多个
4243
task_state="state", # mysql中任务状态字段
@@ -45,5 +46,24 @@ if __name__ == "__main__":
4546
batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24
4647
)
4748

48-
# spider.start_monitor_task() # 下发及监控任务
49-
spider.start() # 采集
49+
parser = ArgumentParser(description="${spider_name}爬虫")
50+
51+
parser.add_argument(
52+
"--start_master",
53+
action="store_true",
54+
help="添加任务",
55+
function=spider.start_monitor_task,
56+
)
57+
parser.add_argument(
58+
"--start_worker", action="store_true", help="启动爬虫", function=spider.start
59+
)
60+
61+
parser.start()
62+
63+
# 直接启动
64+
# spider.start() # 启动爬虫
65+
# spider.start_monitor_task() # 添加任务
66+
67+
# 通过命令行启动
68+
# python ${file_name} --start_master # 添加任务
69+
# python ${file_name} --start_worker # 启动爬虫

‎feapder/templates/task_spider_template.tmpl‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Created on {DATE}
4+
---------
5+
@summary:
6+
---------
7+
@author: {USER}
8+
"""
9+
10+
import feapder
11+
from feapder import ArgumentParser
12+
13+
14+
class ${spider_name}(feapder.TaskSpider):
15+
# 自定义数据库，若项目中有setting.py文件，此自定义可删除
16+
__custom_setting__ = dict(
17+
REDISDB_IP_PORTS="localhost:6379",
18+
REDISDB_USER_PASS="",
19+
REDISDB_DB=0,
20+
MYSQL_IP="localhost",
21+
MYSQL_PORT=3306,
22+
MYSQL_DB="",
23+
MYSQL_USER_NAME="",
24+
MYSQL_USER_PASS="",
25+
)
26+
27+
def start_requests(self, task):
28+
task_id = task.id
29+
url = task.url
30+
yield feapder.Request(url, task_id=task_id)
31+
32+
def parse(self, request, response):
33+
# 提取网站title
34+
print(response.xpath("//title/text()").extract_first())
35+
# 提取网站描述
36+
print(response.xpath("//meta[@name='description']/@content").extract_first())
37+
print("网站地址: ", response.url)
38+
39+
# mysql 需要更新任务状态为做完 即 state=1
40+
yield self.update_task_batch(request.task_id)
41+
42+
43+
if __name__ == "__main__":
44+
# 用mysql做任务表，需要先建好任务任务表
45+
spider = ${spider_name}(
46+
redis_key="xxx:xxx", # 分布式爬虫调度信息存储位置
47+
task_table="", # mysql中的任务表
48+
task_keys=["id", "url"], # 需要获取任务表里的字段名，可添加多个
49+
task_state="state", # mysql中任务状态字段
50+
)
51+
52+
# 用redis做任务表
53+
# spider = ${spider_name}(
54+
# redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置
55+
# task_table="", # 任务表名
56+
# task_table_type="redis", # 任务表类型为redis
57+
# )
58+
59+
parser = ArgumentParser(description="${spider_name}爬虫")
60+
61+
parser.add_argument(
62+
"--start_master",
63+
action="store_true",
64+
help="添加任务",
65+
function=spider.start_monitor_task,
66+
)
67+
parser.add_argument(
68+
"--start_worker", action="store_true", help="启动爬虫", function=spider.start
69+
)
70+
71+
parser.start()
72+
73+
# 直接启动
74+
# spider.start() # 启动爬虫
75+
# spider.start_monitor_task() # 添加任务
76+
77+
# 通过命令行启动
78+
# python ${file_name} --start_master # 添加任务
79+
# python ${file_name} --start_worker # 启动爬虫

‎feapder/templates/update_item_template.tmpl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# -*- coding: utf-8 -*-
2+
"""
3+
Created on {DATE}
4+
---------
5+
@summary:
6+
---------
7+
@author: {USER}
8+
"""
9+
10+
from feapder import UpdateItem
11+
12+
13+
class ${item_name}Item(UpdateItem):
14+
"""
15+
This class was generated by feapder
16+
command: feapder create -i ${command}
17+
"""
18+
19+
__table_name__ = "${table_name}"
20+
21+
def __init__(self, *args, **kwargs):
22+
${propertys}

0 commit comments

Comments
 (0)