现有流程加入支持 GIt 仓库类型的上游-patch-tracking-AtomGit

Mpatch_tracking/api/tracking.py+16-15

	@@ -22,10 +22,14 @@ def delete():
22	input_params = request.args	22	input_params = request.args
23	keys = list(input_params.keys())	23	keys = list(input_params.keys())
24		24
		25	+ param_error = False
25	if not keys or "repo" not in keys:	26	if not keys or "repo" not in keys:
26	- return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)	27	+ param_error = True
27		28
28	if len(set(keys) - {"repo", "branch"}) != 0:	29	if len(set(keys) - {"repo", "branch"}) != 0:
		30	+ param_error = True
		31	+
		32	+ if param_error:
29	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)	33	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
30		34
31	try:	35	try:
	@@ -35,19 +39,16 @@ def delete():
35	delete_tracking(input_params['repo'], input_params['branch'])	39	delete_tracking(input_params['repo'], input_params['branch'])
36	logger.info('Delete tracking repo: %s, branch: %s', input_params['repo'], input_params['branch'])	40	logger.info('Delete tracking repo: %s, branch: %s', input_params['repo'], input_params['branch'])
37	return ResponseCode.ret_message(code=ResponseCode.SUCCESS)	41	return ResponseCode.ret_message(code=ResponseCode.SUCCESS)
38	- else:	42	+
39	- logger.info(	43	+ logger.info('Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch'])
40	- 'Delete tracking repo: %s, branch: %s not found.', input_params['repo'], input_params['branch']	44	+ return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND)
41	- )	45	+ if Tracking.query.filter(Tracking.repo == input_params['repo']).first():
42	- return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND)	46	+ delete_tracking(input_params['repo'])
43	- else:	47	+ logger.info('Delete tracking repo: %s', input_params['repo'])
44	- if Tracking.query.filter(Tracking.repo == input_params['repo']).first():	48	+ return ResponseCode.ret_message(code=ResponseCode.SUCCESS)
45	- delete_tracking(input_params['repo'])	49	+
46	- logger.info('Delete tracking repo: %s', input_params['repo'])	50	+ logger.info('Delete tracking repo: %s not found.', input_params['repo'])
47	- return ResponseCode.ret_message(code=ResponseCode.SUCCESS)	51	+ return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND)
48	- else:
49	- logger.info('Delete tracking repo: %s not found.', input_params['repo'])
50	- return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_NOT_FOUND)
51	except SQLAlchemyError as err:	52	except SQLAlchemyError as err:
52	return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_ERROR, data=err)	53	return ResponseCode.ret_message(code=ResponseCode.DELETE_DB_ERROR, data=err)
53		54
	@@ -98,7 +99,7 @@ def post():
98	if len(required_params) > 1 or (len(required_params) == 1 and required_params[0] != 'scm_commit'):	99	if len(required_params) > 1 or (len(required_params) == 1 and required_params[0] != 'scm_commit'):
99	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)	100	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
100		101
101	- if data['version_control'] != 'github':	102	+ if data['version_control'] not in ["github", "git"]:
102	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)	103	return ResponseCode.ret_message(ResponseCode.INPUT_PARAMETERS_ERROR)
103		104
104	track = Tracking.query.filter_by(repo=data['repo'], branch=data['branch']).first()	105	track = Tracking.query.filter_by(repo=data['repo'], branch=data['branch']).first()

Mpatch_tracking/app.py+2-20

	@@ -9,7 +9,6 @@ from patch_tracking.api.issue import issue
9	from patch_tracking.api.tracking import tracking	9	from patch_tracking.api.tracking import tracking
10	from patch_tracking.database import db	10	from patch_tracking.database import db
11	from patch_tracking.task import task	11	from patch_tracking.task import task
12	-from patch_tracking.util import github_api, gitee_api
13		12
14	logging.config.fileConfig('logging.conf', disable_existing_loggers=False)	13	logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
15		14
	@@ -17,22 +16,6 @@ app = Flask(__name__)
17	logger = logging.getLogger(__name__)	16	logger = logging.getLogger(__name__)
18		17
19		18
20	-def check_token():
21	- """ check gitee/github token """
22	- gitee_token = app.config['GITEE_ACCESS_TOKEN']
23	- github_token = app.config['GITHUB_ACCESS_TOKEN']
24	-
25	- github_ret = github_api.get_user_info(github_token)
26	- if not github_ret[0]:
27	- logger.error(github_ret[1])
28	- sys.exit(1)
29	-
30	- gitee_ret = gitee_api.get_user_info(gitee_token)
31	- if not gitee_ret[0]:
32	- logger.error(gitee_ret[1])
33	- sys.exit(1)
34	-
35	-
36	def check_listen(listen_param):	19	def check_listen(listen_param):
37	""" check LISTEN """	20	""" check LISTEN """
38	check_ret = True	21	check_ret = True
	@@ -56,7 +39,7 @@ def check_settings_conf():
56	check settings.conf	39	check settings.conf
57	"""	40	"""
58	setting_error = False	41	setting_error = False
59	- required_settings = ['LISTEN', 'GITHUB_ACCESS_TOKEN', 'GITEE_ACCESS_TOKEN', 'SCAN_DB_INTERVAL', 'USER', 'PASSWORD']	42	+ required_settings = ['LISTEN', 'GITEE_ACCESS_TOKEN', 'SCAN_DB_INTERVAL', 'USER', 'PASSWORD']
60	for setting in required_settings:	43	for setting in required_settings:
61	if setting in app.config:	44	if setting in app.config:
62	if app.config[setting] == "":	45	if app.config[setting] == "":
	@@ -94,11 +77,10 @@ except (SyntaxError, NameError):
94	logger.error('settings.conf content format error.')	77	logger.error('settings.conf content format error.')
95	sys.exit(1)	78	sys.exit(1)
96		79
97	-check_token()
98	-
99	app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite?check_same_thread=False&timeout=30'	80	app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite?check_same_thread=False&timeout=30'
100	app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False	81	app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
101	app.config['SCHEDULER_EXECUTORS'] = {'default': {'type': 'threadpool', 'max_workers': 100}}	82	app.config['SCHEDULER_EXECUTORS'] = {'default': {'type': 'threadpool', 'max_workers': 100}}
		83	+app.config['GIT_BASE_PATH'] = "GIT_REPO_PATH"
102		84
103	app.register_blueprint(issue, url_prefix="/issue")	85	app.register_blueprint(issue, url_prefix="/issue")
104	app.register_blueprint(tracking, url_prefix="/tracking")	86	app.register_blueprint(tracking, url_prefix="/tracking")

Mpatch_tracking/task/task.py+61-5

	@@ -5,16 +5,48 @@ import datetime
5	import logging	5	import logging
6	from patch_tracking.task import scheduler	6	from patch_tracking.task import scheduler
7	from patch_tracking.database.models import Tracking	7	from patch_tracking.database.models import Tracking
8	-from patch_tracking.util.github_api import GitHubApi	8	+from patch_tracking.util.upstream.github import GitHub
		9	+from patch_tracking.util.upstream.git import Git
9	from patch_tracking.api.business import update_tracking	10	from patch_tracking.api.business import update_tracking
		11	+from patch_tracking.util.upstream.github import get_user_info as github_get_user_info
		12	+from patch_tracking.util.gitee_api import get_user_info as gitee_get_user_info
10		13
11	logger = logging.getLogger(__name__)	14	logger = logging.getLogger(__name__)
12		15
13		16
		17	+def check_token(app):
		18	+ """ check gitee/github token """
		19	+ gitee_token = app.config['GITEE_ACCESS_TOKEN']
		20	+ github_token = app.config['GITHUB_ACCESS_TOKEN']
		21	+ token_error = False
		22	+ try:
		23	+ github_ret = github_get_user_info(github_token)
		24	+ if not github_ret[0]:
		25	+ logger.error(github_ret[1])
		26	+ logger.error('github token is bad credentials.')
		27	+ token_error = True
		28	+ except UnicodeEncodeError:
		29	+ logger.error('github token is bad credentials.')
		30	+ token_error = True
		31	+
		32	+ gitee_ret = gitee_get_user_info(gitee_token)
		33	+ if not gitee_ret[0]:
		34	+ logger.error(gitee_ret[1])
		35	+ logger.error('gitee token is bad credentials.')
		36	+ token_error = True
		37	+
		38	+ if token_error:
		39	+ return False
		40	+ return True
		41	+
		42	+
14	def init(app):	43	def init(app):
15	"""	44	"""
16	scheduler jobs init	45	scheduler jobs init
17	"""	46	"""
		47	+ if not check_token(app):
		48	+ logger.error('[Patch Tracking] Token Error. Stop tracking task.')
		49	+ return
18	scan_db_interval = app.config['SCAN_DB_INTERVAL']	50	scan_db_interval = app.config['SCAN_DB_INTERVAL']
19	scheduler.init_app(app)	51	scheduler.init_app(app)
20	scheduler.add_job(	52	scheduler.add_job(
	@@ -55,13 +87,35 @@ def check_empty_commit_id(flask_app):
55	"""	87	"""
56	with flask_app.app_context():	88	with flask_app.app_context():
57	new_track = get_track_from_db()	89	new_track = get_track_from_db()
58	- github_api = GitHubApi()
59	for item in new_track:	90	for item in new_track:
60	if item.scm_commit:	91	if item.scm_commit:
61	continue	92	continue
62	- status, result = github_api.get_latest_commit(item.scm_repo, item.scm_branch)	93	+ if item.version_control == "github":
63	- if status == 'success':	94	+ github_api = GitHub(item)
64	- commit_id = result['latest_commit']	95	+ status, result = github_api.get_latest_commit_id()
		96	+ if status == 'success':
		97	+ commit_id = result['latest_commit']
		98	+ data = {
		99	+ 'version_control': item.version_control,
		100	+ 'repo': item.repo,
		101	+ 'branch': item.branch,
		102	+ 'enabled': item.enabled,
		103	+ 'scm_commit': commit_id,
		104	+ 'scm_branch': item.scm_branch,
		105	+ 'scm_repo': item.scm_repo
		106	+ }
		107	+ update_tracking(data)
		108	+ else:
		109	+ logger.error(
		110	+ 'Check empty CommitID: Fail to get latest commit id of scm_repo: %s scm_branch: %s. \
		111	+ Return val: %s', item.scm_repo, item.scm_branch, result
		112	+ )
		113	+ elif item.version_control == "git":
		114	+ git_api = Git(item)
		115	+ commit_id = git_api.git_latest_sha()
		116	+ if not commit_id:
		117	+ return None
		118	+
65	data = {	119	data = {
66	'version_control': item.version_control,	120	'version_control': item.version_control,
67	'repo': item.repo,	121	'repo': item.repo,
	@@ -78,6 +132,8 @@ def check_empty_commit_id(flask_app):
78	item.scm_repo, item.scm_branch, result	132	item.scm_repo, item.scm_branch, result
79	)	133	)
80		134
		135	+ return None
		136	+
81		137
82	def get_track_from_db():	138	def get_track_from_db():
83	"""	139	"""

Mpatch_tracking/task/task_apscheduler.py+27-102

	@@ -9,11 +9,11 @@ import random
9	from sqlalchemy.exc import SQLAlchemyError	9	from sqlalchemy.exc import SQLAlchemyError
10	from patch_tracking.util.gitee_api import create_branch, upload_patch, create_gitee_issue	10	from patch_tracking.util.gitee_api import create_branch, upload_patch, create_gitee_issue
11	from patch_tracking.util.gitee_api import create_pull_request, get_path_content, upload_spec, create_spec	11	from patch_tracking.util.gitee_api import create_pull_request, get_path_content, upload_spec, create_spec
12	-from patch_tracking.util.github_api import GitHubApi
13	from patch_tracking.database.models import Tracking	12	from patch_tracking.database.models import Tracking
14	from patch_tracking.api.business import update_tracking, create_issue	13	from patch_tracking.api.business import update_tracking, create_issue
15	from patch_tracking.task import scheduler	14	from patch_tracking.task import scheduler
16	from patch_tracking.util.spec import Spec	15	from patch_tracking.util.spec import Spec
		16	+from patch_tracking.util.upstream import Factory
17		17
18	logger = logging.getLogger(__name__)	18	logger = logging.getLogger(__name__)
19		19
	@@ -25,9 +25,10 @@ def upload_patch_to_gitee(track):
25	cur_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")	25	cur_time = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")
26	with scheduler.app.app_context():	26	with scheduler.app.app_context():
27	logger.info('[Patch Tracking %s] track.scm_commit_id: %s.', cur_time, track.scm_commit)	27	logger.info('[Patch Tracking %s] track.scm_commit_id: %s.', cur_time, track.scm_commit)
28	- patch = get_scm_patch(track)	28	+ git_api = Factory.create(track)
		29	+ patch = get_scm_patch(track, git_api)
29	if patch:	30	if patch:
30	- issue = create_patch_issue_pr(patch, cur_time)	31	+ issue = create_patch_issue_pr(patch, cur_time, git_api)
31	if issue:	32	if issue:
32	create_issue_db(issue)	33	create_issue_db(issue)
33	else:	34	else:
	@@ -36,50 +37,12 @@ def upload_patch_to_gitee(track):
36	logger.debug('[Patch Tracking %s] No new commit.', cur_time)	37	logger.debug('[Patch Tracking %s] No new commit.', cur_time)
37		38
38		39
39	-def get_all_commit_info(scm_repo, db_commit, latest_commit):	40	+def get_scm_patch(track, git_api):
40	- """
41	- get all commit information between to commits
42	- """
43	- commit_list = list()
44	- github_api = GitHubApi()
45	-
46	- while db_commit != latest_commit:
47	- status, result = github_api.get_commit_info(scm_repo, latest_commit)
48	- logger.debug('get_commit_info: %s %s', status, result)
49	- if status == 'success':
50	- if 'parent' in result:
51	- ret = github_api.get_patch(scm_repo, latest_commit, latest_commit)
52	- logger.debug('get patch api ret: %s', ret)
53	- if ret['status'] == 'success':
54	- result['patch_content'] = ret['api_ret']
55	- # inverted insert commit_list
56	- commit_list.insert(0, result)
57	- else:
58	- logger.error('Get scm: %s commit: %s patch failed. Result: %s', scm_repo, latest_commit, result)
59	-
60	- latest_commit = result['parent']
61	- else:
62	- logger.info(
63	- '[Patch Tracking] Successful get scm commit from %s to %s ID/message/time/patch.', db_commit,
64	- latest_commit
65	- )
66	- break
67	- else:
68	- logger.error(
69	- '[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', scm_repo, latest_commit,
70	- result
71	- )
72	-
73	- return commit_list
74	-
75	-
76	-def get_scm_patch(track):
77	"""	41	"""
78	Traverse the Tracking data table to get the patch file of enabled warehouse.	42	Traverse the Tracking data table to get the patch file of enabled warehouse.
79	Different warehouse has different acquisition methods	43	Different warehouse has different acquisition methods
80	:return:	44	:return:
81	"""	45	"""
82	- github_api = GitHubApi()
83	scm_dict = dict(	46	scm_dict = dict(
84	scm_repo=track.scm_repo,	47	scm_repo=track.scm_repo,
85	scm_branch=track.scm_branch,	48	scm_branch=track.scm_branch,
	@@ -89,75 +52,40 @@ def get_scm_patch(track):
89	branch=track.branch,	52	branch=track.branch,
90	version_control=track.version_control	53	version_control=track.version_control
91	)	54	)
92	- status, result = github_api.get_latest_commit(scm_dict['scm_repo'], scm_dict['scm_branch'])
93	- logger.debug(
94	- 'repo: %s branch: %s. get_latest_commit: %s %s', scm_dict['scm_repo'], scm_dict['scm_branch'], status, result
95	- )
96		55
97	- if status == 'success':	56	+ commit_list = git_api.get_scm_patch()
98	- commit_id = result['latest_commit']	57	+ if commit_list:
99	- if not scm_dict['scm_commit']:	58	+ scm_dict['commit_list'] = commit_list
100	- data = {	59	+ return scm_dict
101	- 'version_control': scm_dict['version_control'],	60	+ logger.info('repo: %s branch: %s. get_latest_commit is None.', scm_dict['scm_repo'], scm_dict['scm_branch'])
102	- 'repo': scm_dict['repo'],	61	+
103	- 'branch': scm_dict['branch'],
104	- 'enabled': scm_dict['enabled'],
105	- 'scm_commit': commit_id,
106	- 'scm_branch': scm_dict['scm_branch'],
107	- 'scm_repo': scm_dict['scm_repo']
108	- }
109	- update_tracking(data)
110	- logger.info(
111	- '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: None.',
112	- scm_dict['scm_repo'], scm_dict['scm_branch'], result['latest_commit']
113	- )
114	- else:
115	- if commit_id != scm_dict['scm_commit']:
116	- commit_list = get_all_commit_info(scm_dict['scm_repo'], scm_dict['scm_commit'], commit_id)
117	- scm_dict['commit_list'] = commit_list
118	- return scm_dict
119	- logger.info(
120	- '[Patch Tracking] Scm_repo: %s Scm_branch: %s.Get latest commit ID: %s From commit ID: %s. '
121	- 'Nothing need to do.', scm_dict['scm_repo'], scm_dict['scm_branch'], commit_id, scm_dict['scm_commit']
122	- )
123	- else:
124	- logger.error(
125	- '[Patch Tracking] Fail to get latest commit id of scm_repo: %s scm_branch: %s. Return val: %s',
126	- scm_dict['scm_repo'], scm_dict['scm_branch'], result
127	- )
128	return None	62	return None
129		63
130		64
131	-def create_patch_issue_pr(patch, cur_time):	65	+def create_patch_issue_pr(patch, cur_time, git_api):
132	"""	66	"""
133	Create temporary branches, submit files, and create PR and issue	67	Create temporary branches, submit files, and create PR and issue
134	:return:	68	:return:
135	"""	69	"""
136	issue_dict = dict()	70	issue_dict = dict()
137	- if not patch:	71	+ gitee_repo = patch["repo"].replace("https://gitee.com/", "")
138	- return None
139	-
140	issue_dict['repo'] = patch['repo']	72	issue_dict['repo'] = patch['repo']
141	issue_dict['branch'] = patch['branch']	73	issue_dict['branch'] = patch['branch']
142	new_branch = 'patch-tracking/' + cur_time	74	new_branch = 'patch-tracking/' + cur_time
143	- result = create_branch(patch['repo'], patch['branch'], new_branch)	75	+ result = create_branch(gitee_repo, patch['branch'], new_branch)
144	if result == 'success':	76	if result == 'success':
145	logger.info('[Patch Tracking %s] Successful create branch: %s', cur_time, new_branch)	77	logger.info('[Patch Tracking %s] Successful create branch: %s', cur_time, new_branch)
146	else:	78	else:
147	logger.error('[Patch Tracking %s] Fail to create branch: %s', cur_time, new_branch)	79	logger.error('[Patch Tracking %s] Fail to create branch: %s', cur_time, new_branch)
148	return None	80	return None
149	patch_lst = list()	81	patch_lst = list()
150	- issue_table = "\| Commit \| Datetime \| Message \|\n\| ------ \| ------ \| ------ \|\n"
151	for latest_commit in patch['commit_list']:	82	for latest_commit in patch['commit_list']:
152	scm_commit_url = '/'.join(['https://github.com', patch['scm_repo'], 'commit', latest_commit['commit_id']])	83	scm_commit_url = '/'.join(['https://github.com', patch['scm_repo'], 'commit', latest_commit['commit_id']])
153	latest_commit['message'] = latest_commit['message'].replace("\r", "").replace("\n", "<br>")	84	latest_commit['message'] = latest_commit['message'].replace("\r", "").replace("\n", "<br>")
154	- issue_table += '\| [{}]({}) \| {} \| {} \|'.format(
155	- latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message']
156	- ) + '\n'
157		85
158	patch_file_content = latest_commit['patch_content']	86	patch_file_content = latest_commit['patch_content']
159	post_data = {	87	post_data = {
160	- 'repo': patch['repo'],	88	+ 'repo': gitee_repo,
161	'branch': new_branch,	89	'branch': new_branch,
162	'latest_commit_id': latest_commit['commit_id'],	90	'latest_commit_id': latest_commit['commit_id'],
163	'patch_file_content': str(patch_file_content),	91	'patch_file_content': str(patch_file_content),
	@@ -183,25 +111,22 @@ def create_patch_issue_pr(patch, cur_time):
183	logger.error('[Patch Tracking %s] Fail to upload spec file. Result: %s', cur_time, result)	111	logger.error('[Patch Tracking %s] Fail to upload spec file. Result: %s', cur_time, result)
184	return None	112	return None
185		113
		114	+ issue_table = git_api.issue_table(patch['commit_list'])
186	logger.debug(issue_table)	115	logger.debug(issue_table)
187	- result = create_gitee_issue(patch['repo'], patch['branch'], issue_table, cur_time)	116	+ result = create_gitee_issue(gitee_repo, patch['branch'], issue_table, cur_time)
188	if result[0] == 'success':	117	if result[0] == 'success':
189	issue_num = result[1]	118	issue_num = result[1]
190	logger.info('[Patch Tracking %s] Successfully create issue: %s', cur_time, issue_num)	119	logger.info('[Patch Tracking %s] Successfully create issue: %s', cur_time, issue_num)
191		120
192	retry_count = 10	121	retry_count = 10
193	while retry_count > 0:	122	while retry_count > 0:
194	- ret = create_pull_request(patch['repo'], patch['branch'], new_branch, issue_num, cur_time)	123	+ ret = create_pull_request(gitee_repo, patch['branch'], new_branch, issue_num, cur_time)
195	if ret == 'success':	124	if ret == 'success':
196	logger.info('[Patch Tracking %s] Successfully create PR of issue: %s.', cur_time, issue_num)	125	logger.info('[Patch Tracking %s] Successfully create PR of issue: %s.', cur_time, issue_num)
197	break	126	break
198	- else:	127	+ logger.warning('[Patch Tracking %s] Fail to create PR of issue: %s. Result: %s', cur_time, issue_num, ret)
199	- logger.warning(	128	+ retry_count -= 1
200	- '[Patch Tracking %s] Fail to create PR of issue: %s. Result: %s', cur_time, issue_num, ret	129	+ time.sleep(random.random() * 5)
201	- )
202	- retry_count -= 1
203	- time.sleep(random.random() * 2)
204	- continue
205	if retry_count == 0:	130	if retry_count == 0:
206	logger.error('[Patch Tracking %s] Fail to create PR of issue: %s.', cur_time, issue_num)	131	logger.error('[Patch Tracking %s] Fail to create PR of issue: %s.', cur_time, issue_num)
207	return None	132	return None
	@@ -233,25 +158,25 @@ def upload_spec_to_repo(patch, patch_lst, cur_time):
233	update and upload spec file	158	update and upload spec file
234	"""	159	"""
235	new_branch = 'patch-tracking/' + cur_time	160	new_branch = 'patch-tracking/' + cur_time
236	-	161	+ gitee_repo = patch["repo"].replace("https://gitee.com/", "")
237	- _, repo_name = patch['repo'].split('/')	162	+ _, repo_name = gitee_repo.split('/')
238	spec_file = repo_name + '.spec'	163	spec_file = repo_name + '.spec'
239		164
240	patch_file_lst = [patch + '.patch' for patch in patch_lst]	165	patch_file_lst = [patch + '.patch' for patch in patch_lst]
241		166
242	- log_title = "{} patch-tracking".format(cur_time)	167	+ log_title = "{} patch-tracking".format(datetime.datetime.now().strftime("%a %b %d %Y"))
243	log_content = "append patch file of upstream repository from <{}> to <{}>".format(patch_lst[0], patch_lst[-1])	168	log_content = "append patch file of upstream repository from <{}> to <{}>".format(patch_lst[0], patch_lst[-1])
244		169
245	- ret = get_path_content(patch['repo'], patch['branch'], spec_file)	170	+ ret = get_path_content(gitee_repo, patch['branch'], spec_file)
246	if 'content' in ret:	171	if 'content' in ret:
247	spec_content = str(base64.b64decode(ret['content']), encoding='utf-8')	172	spec_content = str(base64.b64decode(ret['content']), encoding='utf-8')
248	spec_sha = ret['sha']	173	spec_sha = ret['sha']
249	new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)	174	new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)
250	- result = update_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec, spec_sha)	175	+ result = update_spec_to_repo(gitee_repo, new_branch, cur_time, new_spec, spec_sha)
251	else:	176	else:
252	spec_content = ''	177	spec_content = ''
253	new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)	178	new_spec = modify_spec(log_title, log_content, patch_file_lst, spec_content)
254	- result = create_spec_to_repo(patch['repo'], new_branch, cur_time, new_spec)	179	+ result = create_spec_to_repo(gitee_repo, new_branch, cur_time, new_spec)
255		180
256	return result	181	return result
257		182

Mpatch_tracking/util/gitee_api.py+1-1

	@@ -72,7 +72,7 @@ def get_user_info(token):
72	if ret:	72	if ret:
73	if ret_info.status_code == 200:	73	if ret_info.status_code == 200:
74	return True, ret_info.text	74	return True, ret_info.text
75	- return False, ret_info.json()	75	+ return False, ret_info.text
76		76
77	return False, ret_info	77	return False, ret_info
78		78

Dpatch_tracking/util/github_api.py+0-156

	@@ -1,156 +0,0 @@
1	-"""
2	-functionality of invoking GitHub API
3	-"""
4	-import time
5	-import logging
6	-import requests
7	-from requests import exceptions
8	-from flask import current_app
9	-
10	-logger = logging.getLogger(__name__)
11	-
12	-
13	-def get_user_info(token):
14	- """
15	- get user info
16	- """
17	- url = "https://api.github.com/user"
18	- count = 30
19	- token = 'token ' + token
20	- headers = {
21	- 'User-Agent': 'Mozilla/5.0',
22	- 'Authorization': token,
23	- 'Content-Type': 'application/json',
24	- 'Connection': 'close',
25	- 'method': 'GET',
26	- 'Accept': 'application/json'
27	- }
28	- while count > 0:
29	- try:
30	- ret = requests.get(url, headers=headers)
31	- if ret.status_code == 200:
32	- return True, ret.text
33	- return False, ret.json()
34	- except exceptions.ConnectionError as err:
35	- logger.warning(err)
36	- time.sleep(10)
37	- count -= 1
38	- continue
39	- except UnicodeEncodeError:
40	- return False, 'github token is bad credentials.'
41	- except IOError as error:
42	- return False, error
43	- if count == 0:
44	- logger.error('Fail to connnect to github: %s after retry 30 times.', url)
45	- return False, 'connect error'
46	-
47	-
48	-class GitHubApi(object):
49	- """
50	- Encapsulates GitHub functionality
51	- """
52	- def __init__(self):
53	- github_token = current_app.config['GITHUB_ACCESS_TOKEN']
54	- token = 'token ' + github_token
55	- self.headers = {
56	- 'User-Agent': 'Mozilla/5.0',
57	- 'Authorization': token,
58	- 'Content-Type': 'application/json',
59	- 'Connection': 'close',
60	- 'method': 'GET',
61	- 'Accept': 'application/json'
62	- }
63	-
64	- def api_request(self, url):
65	- """
66	- request GitHub API
67	- """
68	- logger.debug("Connect url: %s", url)
69	- count = 30
70	- while count > 0:
71	- try:
72	- response = requests.get(url, headers=self.headers)
73	- return response
74	- except exceptions.ConnectionError as err:
75	- logger.warning(err)
76	- time.sleep(10)
77	- count -= 1
78	- continue
79	- except IOError as err:
80	- logger.error(err)
81	- return False
82	- if count == 0:
83	- logger.error('Fail to connnect to github: %s after retry 30 times.', url)
84	- return False
85	-
86	- def get_commit_info(self, repo_url, commit_id):
87	- """
88	- get commit info
89	- """
90	- res_dict = dict()
91	- api_url = 'https://api.github.com/repos'
92	- url = '/'.join([api_url, repo_url, 'commits', commit_id])
93	- ret = self.api_request(url)
94	- if ret:
95	- if ret.status_code == 200:
96	- res_dict['commit_id'] = commit_id
97	- res_dict['message'] = ret.json()['commit']['message']
98	- res_dict['time'] = ret.json()['commit']['author']['date']
99	- if 'parents' in ret.json() and ret.json()['parents']:
100	- res_dict['parent'] = ret.json()['parents'][0]['sha']
101	- return 'success', res_dict
102	-
103	- logger.error('%s failed. Return val: %s', url, ret)
104	- return 'error', ret.json()
105	- return 'error', 'connect error'
106	-
107	- def get_latest_commit(self, repo_url, branch):
108	- """
109	- get latest commit_ID, commit_message, commit_date
110	- :param repo_url:
111	- :param branch:
112	- :return: res_dict
113	- """
114	- api_url = 'https://api.github.com/repos'
115	- url = '/'.join([api_url, repo_url, 'branches', branch])
116	- ret = self.api_request(url)
117	- res_dict = dict()
118	- if ret:
119	- if ret.status_code == 200:
120	- res_dict['latest_commit'] = ret.json()['commit']['sha']
121	- res_dict['message'] = ret.json()['commit']['commit']['message']
122	- res_dict['time'] = ret.json()['commit']['commit']['committer']['date']
123	- return 'success', res_dict
124	-
125	- logger.error('%s failed. Return val: %s', url, ret)
126	- return 'error', ret.json()
127	-
128	- return 'error', 'connect error'
129	-
130	- def get_patch(self, repo_url, scm_commit, last_commit):
131	- """
132	- get patch
133	- """
134	- api_url = 'https://github.com'
135	- if scm_commit != last_commit:
136	- commit = scm_commit + '...' + last_commit + '.diff'
137	- else:
138	- commit = scm_commit + '^...' + scm_commit + '.diff'
139	- ret_dict = dict()
140	-
141	- url = '/'.join([api_url, repo_url, 'compare', commit])
142	- ret = self.api_request(url)
143	- if ret:
144	- if ret.status_code == 200:
145	- patch_content = ret.text
146	- ret_dict['status'] = 'success'
147	- ret_dict['api_ret'] = patch_content
148	- else:
149	- logger.error('%s failed. Return val: %s', url, ret)
150	- ret_dict['status'] = 'error'
151	- ret_dict['api_ret'] = ret.text
152	- else:
153	- ret_dict['status'] = 'error'
154	- ret_dict['api_ret'] = 'fail to connect github by api.'
155	-
156	- return ret_dict

Mpatch_tracking/util/upstream/git.py+5-5

	@@ -2,10 +2,10 @@
2	import logging	2	import logging
3	import os	3	import os
4	from datetime import datetime	4	from datetime import datetime
5	-from flask import current_app
6	-from sqlalchemy.exc import SQLAlchemyError
7	import git	5	import git
8	import git.exc	6	import git.exc
		7	+from flask import current_app
		8	+from sqlalchemy.exc import SQLAlchemyError
9	from patch_tracking.api.business import update_tracking	9	from patch_tracking.api.business import update_tracking
10	import patch_tracking.util.upstream.upstream as upstream	10	import patch_tracking.util.upstream.upstream as upstream
11		11
	@@ -78,14 +78,14 @@ class Git(upstream.Upstream):
78	get latest commit id	78	get latest commit id
79	"""	79	"""
80	repo_path = os.path.join(self.base_path, self.repo_dir_name)	80	repo_path = os.path.join(self.base_path, self.repo_dir_name)
81	- logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.branch)	81	+ logging.info("Getting latest commit id of repo: %s branch: %s .", repo_path, self.track.scm_branch)
82	try:	82	try:
83	repo = git.Repo(repo_path)	83	repo = git.Repo(repo_path)
84	- sha = repo.commit(self.track.branch).hexsha	84	+ sha = repo.commit(self.track.scm_branch).hexsha
85	return sha	85	return sha
86	except git.exc.GitError as err:	86	except git.exc.GitError as err:
87	logging.error(	87	logging.error(
88	- "Get latest commit id of repo: %s branch: %s failed. Error: %s", repo_path, self.track.branch, err	88	+ "Get latest commit id of repo: %s branch: %s failed. Error: %s", repo_path, self.track.scm_branch, err
89	)	89	)
90	return False	90	return False
91		91

Mpatch_tracking/util/upstream/github.py+134-17

	@@ -1,7 +1,9 @@
1	"""github upstream"""	1	"""github upstream"""
2	import logging	2	import logging
		3	+import time
		4	+import requests
3	from flask import current_app	5	from flask import current_app
4	-from patch_tracking.util.github_api import GitHubApi	6	+from requests import exceptions
5	from patch_tracking.api.business import update_tracking	7	from patch_tracking.api.business import update_tracking
6	from sqlalchemy.exc import SQLAlchemyError	8	from sqlalchemy.exc import SQLAlchemyError
7	import patch_tracking.util.upstream.upstream as upstream	9	import patch_tracking.util.upstream.upstream as upstream
	@@ -9,34 +11,151 @@ import patch_tracking.util.upstream.upstream as upstream
9	logger = logging.getLogger(__name__)	11	logger = logging.getLogger(__name__)
10		12
11		13
		14	+def get_user_info(token):
		15	+ """
		16	+ get user info
		17	+ """
		18	+ url = "https://api.github.com/user"
		19	+ count = 30
		20	+ token = 'token ' + token
		21	+ headers = {
		22	+ 'User-Agent': 'Mozilla/5.0',
		23	+ 'Authorization': token,
		24	+ 'Content-Type': 'application/json',
		25	+ 'Connection': 'close',
		26	+ 'method': 'GET',
		27	+ 'Accept': 'application/json'
		28	+ }
		29	+ while count > 0:
		30	+ try:
		31	+ ret = requests.get(url, headers=headers)
		32	+ if ret.status_code == 200:
		33	+ return True, ret.text
		34	+ return False, ret.text
		35	+ except exceptions.ConnectionError as err:
		36	+ logger.warning(err)
		37	+ time.sleep(10)
		38	+ count -= 1
		39	+ continue
		40	+ except UnicodeEncodeError:
		41	+ return False, 'github token is bad credentials.'
		42	+ except IOError as error:
		43	+ return False, error
		44	+ if count == 0:
		45	+ logger.error('Fail to connnect to github: %s after retry 30 times.', url)
		46	+ return False, 'connect error'
		47	+
		48	+
12	class GitHub(upstream.Upstream):	49	class GitHub(upstream.Upstream):
13	"""	50	"""
14	GitHub type	51	GitHub type
15	"""	52	"""
16	def __init__(self, track):	53	def __init__(self, track):
17	super().__init__(track)	54	super().__init__(track)
18	- self.github_api = GitHubApi()	55	+ _token = 'token ' + current_app.config['GITHUB_ACCESS_TOKEN']
19	- self.token = current_app.config['GITHUB_ACCESS_TOKEN']	56	+ self.headers = {
		57	+ 'User-Agent': 'Mozilla/5.0',
		58	+ 'Authorization': _token,
		59	+ 'Content-Type': 'application/json',
		60	+ 'Connection': 'close',
		61	+ 'method': 'GET',
		62	+ 'Accept': 'application/json'
		63	+ }
		64	+
		65	+ def api_request(self, url):
		66	+ """
		67	+ request GitHub API
		68	+ """
		69	+ logger.debug("Connect url: %s", url)
		70	+ count = 30
		71	+ while count > 0:
		72	+ try:
		73	+ response = requests.get(url, headers=self.headers)
		74	+ return response
		75	+ except exceptions.ConnectionError as err:
		76	+ logger.warning(err)
		77	+ time.sleep(10)
		78	+ count -= 1
		79	+ continue
		80	+ except IOError as err:
		81	+ logger.error(err)
		82	+ return False
		83	+ if count == 0:
		84	+ logger.error('Fail to connnect to github: %s after retry 30 times.', url)
		85	+ return False
		86	+
		87	+ def get_patch(self, repo_url, scm_commit, last_commit):
		88	+ """
		89	+ get patch
		90	+ """
		91	+ api_url = 'https://github.com'
		92	+ if scm_commit != last_commit:
		93	+ commit = scm_commit + '...' + last_commit + '.diff'
		94	+ else:
		95	+ commit = scm_commit + '^...' + scm_commit + '.diff'
		96	+ ret_dict = dict()
		97	+
		98	+ url = '/'.join([api_url, repo_url, 'compare', commit])
		99	+ ret = self.api_request(url)
		100	+ if ret:
		101	+ if ret.status_code == 200:
		102	+ patch_content = ret.text
		103	+ ret_dict['status'] = 'success'
		104	+ ret_dict['api_ret'] = patch_content
		105	+ else:
		106	+ logger.error('%s failed. Return val: %s', url, ret)
		107	+ ret_dict['status'] = 'error'
		108	+ ret_dict['api_ret'] = ret.text
		109	+ else:
		110	+ ret_dict['status'] = 'error'
		111	+ ret_dict['api_ret'] = 'fail to connect github by api.'
		112	+ return ret_dict
20		113
21	def get_latest_commit_id(self):	114	def get_latest_commit_id(self):
22	"""	115	"""
23	- get latest commit id	116	+ get latest commit_ID, commit_message, commit_date
		117	+ :return: res_dict
24	"""	118	"""
25	- status, result = self.github_api.get_latest_commit(self.track.scm_repo, self.track.scm_branch)	119	+ api_url = 'https://api.github.com/repos'
26	- logger.debug(	120	+ url = '/'.join([api_url, self.track.scm_repo, 'branches', self.track.scm_branch])
27	- 'repo: %s branch: %s. get_latest_commit: %s %s', self.track.scm_repo, self.track.scm_branch, status, result	121	+ ret = self.api_request(url)
28	- )	122	+ res_dict = dict()
		123	+ if ret:
		124	+ if ret.status_code == 200:
		125	+ res_dict['latest_commit'] = ret.json()['commit']['sha']
		126	+ res_dict['message'] = ret.json()['commit']['commit']['message']
		127	+ res_dict['time'] = ret.json()['commit']['commit']['committer']['date']
		128	+ logger.debug(
		129	+ 'repo: %s branch: %s. get_latest_commit: %s %s', self.track.scm_repo, self.track.scm_branch,
		130	+ 'success', res_dict
		131	+ )
		132	+ return 'success', res_dict
29		133
30	- return status, result	134	+ logger.error('%s failed. Return val: %s', url, ret)
		135	+ return 'error', ret.json()
		136	+ return 'error', 'connect error'
31		137
32	- def get_commit_info(self, commit_id):	138	+ def get_commit_info(self, repo_url, commit_id):
33	"""	139	"""
34	get commit info	140	get commit info
35	"""	141	"""
36	- status, result = self.github_api.get_commit_info(self.track.scm_repo, commit_id)	142	+ res_dict = dict()
37	- logger.debug('get_commit_info: %s %s', status, result)	143	+ api_url = 'https://api.github.com/repos'
		144	+ url = '/'.join([api_url, repo_url, 'commits', commit_id])
		145	+ ret = self.api_request(url)
		146	+ if ret:
		147	+ if ret.status_code == 200:
		148	+ res_dict['commit_id'] = commit_id
		149	+ res_dict['message'] = ret.json()['commit']['message']
		150	+ res_dict['time'] = ret.json()['commit']['author']['date']
		151	+ if 'parents' in ret.json() and ret.json()['parents']:
		152	+ res_dict['parent'] = ret.json()['parents'][0]['sha']
		153	+ logger.debug('get_commit_info: %s %s', 'success', res_dict)
		154	+ return 'success', res_dict
38		155
39	- return status, result	156	+ logger.error('%s failed. Return val: %s', url, ret)
		157	+ return 'error', ret.json()
		158	+ return 'error', 'connect error'
40		159
41	def get_patch_list(self):	160	def get_patch_list(self):
42	"""	161	"""
	@@ -70,11 +189,11 @@ class GitHub(upstream.Upstream):
70	return None	189	return None
71		190
72	while self.track.scm_commit != latest_commit:	191	while self.track.scm_commit != latest_commit:
73	- status, result = self.get_commit_info(latest_commit)	192	+ status, result = self.get_commit_info(self.track.scm_repo, latest_commit)
74	logger.debug('get_commit_info: %s %s', status, result)	193	logger.debug('get_commit_info: %s %s', status, result)
75	if status == 'success':	194	if status == 'success':
76	if 'parent' in result:	195	if 'parent' in result:
77	- ret = self.github_api.get_patch(self.track.scm_repo, latest_commit, latest_commit)	196	+ ret = self.get_patch(self.track.scm_repo, latest_commit, latest_commit)
78	logger.debug('get patch api ret: %s', ret)	197	logger.debug('get patch api ret: %s', ret)
79	if ret['status'] == 'success':	198	if ret['status'] == 'success':
80	result['patch_content'] = ret['api_ret']	199	result['patch_content'] = ret['api_ret']
	@@ -97,7 +216,6 @@ class GitHub(upstream.Upstream):
97	'[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', self.track.scm_repo,	216	'[Patch Tracking] Get scm: %s commit: %s ID/message/time failed. Result: %s', self.track.scm_repo,
98	latest_commit, result	217	latest_commit, result
99	)	218	)
100	-
101	return commit_list	219	return commit_list
102		220
103	def get_scm_patch(self):	221	def get_scm_patch(self):
	@@ -114,5 +232,4 @@ class GitHub(upstream.Upstream):
114	issue_table += '\| [{}]({}) \| {} \| {} \|'.format(	232	issue_table += '\| [{}]({}) \| {} \| {} \|'.format(
115	latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message']	233	latest_commit['commit_id'][0:7], scm_commit_url, latest_commit['time'], latest_commit['message']
116	) + '\n'	234	) + '\n'
117	-
118	return issue_table	235	return issue_table