【已解决】TLSR825X系列mesh SDK多核编译
Posted: 2022年 Sep 21日 10:27
请问一下，使用TLSR825X系列mesh SDK，如何配置可以实现多核编译。默认的编译太慢了？谢谢
请问一下，使用TLSR825X系列mesh SDK，如何配置可以实现多核编译。默认的编译太慢了？谢谢
我们针对gcc类似的SDK，是基于python，直接调用编译器对.c进行编译，可以使能多线程编译，但是我们发现使能多线程之后，输出的编译log就混杂在一起了，因此对外是用阻塞式单线程编译.c。我们这边接收您的建议，看看如何优化。
目前还发现make编译机制没有效果。我只修改了app_common.c，编译时编译了很多内容。
有个临时的方法，可以试一试：
编辑 .ide_tool/components/my_ide/my_ide_gcc.py
在 # .s to .o 之后加一个延时：
Code: Select all
def tbuild(self):
...
# c to .o
for c_file in self.src['c_files']:
self.__compile('.c',c_file,log_path,evn)
print("[cc] %s"%(c_file))
# .s to .o
for s_file in self.src['s_files']:
self.__compile('.s',s_file,log_path,evn)
print("[cc] %s"%(s_file))
time.sleep(2); #<<<<<<<<<<<<<<<------ 这个是延时2S，因为上面编译.c,.s都是非阻塞并行的，如果不加延时，可能没编译好，就进行链接了，导致链接不成功（这个2S根据您自己的机器，做调整）
在 def __compile(self,kind,file,out_path,evn):
中的 my_exe_simple(cmd,0,evn,self.var_map) 第二参数改为 0 ，表示非阻塞编译 .c,.s
不是基于 makefile，是使用 python 直接对工程进行编译，后续也会加入类似 makefile 的增量编译
beautifulzzzz 2022年 Sep 21日 11:08有个临时的方法，可以试一试：
编辑 .ide_tool/components/my_ide/my_ide_gcc.py在 # .s to .o 之后加一个延时：
Code: Select all
def tbuild(self): ... # c to .o for c_file in self.src['c_files']: self.__compile('.c',c_file,log_path,evn) print("[cc] %s"%(c_file))
# .s to .o for s_file in self.src['s_files']: self.__compile('.s',s_file,log_path,evn) print("[cc] %s"%(s_file)) time.sleep(2); #<<<<<<<<<<<<<<<------ 这个是延时2S，因为上面编译.c,.s都是非阻塞并行的，如果不加延时，可能没编译好，就进行链接了，导致链接不成功（这个2S根据您自己的机器，做调整）
在 def __compile(self,kind,file,out_path,evn):
中的 my_exe_simple(cmd,0,evn,self.var_map) 第二参数改为 0 ，表示非阻塞编译 .c,.s
好的，谢谢
可以参照下面修改下my_ide_gcc.py和my_ide_base.py脚本，可以实现增量编译
Code: Select all
def if_need_rebuild(self,dep_file):
with open(dep_file,"r") as f:
list_line1 = f.readline().split(' ')
if(list_line1[1] != '\\\n'):
try:
if(os.path.getmtime(list_line1[1]) > os.path.getmtime(dep_file)):
return True
except:
return True
for line in f.readlines():
line = line.lstrip()
line = line.strip('\n')
list_n = line.split(' ')
for file in list_n:
if(file == '\\'):
break
try:
if(os.path.getmtime(file) > os.path.getmtime(dep_file)):
return True
except:
print("file not exit")
return True
return False
def __compile(self,kind,file,out_path,evn):
cc = self.cmd['gcc']['cc']
asm = self.cmd['gcc']['asm']
c_flags = self.cmd['gcc']['c_flags']
c_macros = self.cmd['gcc']['c_macros']
s_flags = self.cmd['gcc']['s_flags']
o_file = out_path+'/'+os.path.splitext(os.path.basename(file))[0]+'.o'
d_file = o_file+'.d'
if kind == '.c':
gcc_h_file = out_path+'/'+'gcc_h.txt'
Note=open(gcc_h_file,mode='w')
Note.write(self.src['h_dir_str'])
Note.close()
if((os.path.exists(d_file) == True) and (self.if_need_rebuild(d_file) == False)):
return o_file
if (os.path.exists(o_file)):
my_file_rm_file(o_file)
cmd = "%s %s @%s -c %s -o %s %s -MMD -MF %s"%(cc,c_flags,gcc_h_file,file,o_file,c_macros, d_file)
elif kind == '.s':
if (os.path.exists(o_file)):
my_file_rm_file(o_file)
cmd = "%s %s -c %s -o %s"%(asm,s_flags,file,o_file)
my_exe_simple(cmd,1,evn,self.var_map)
return o_fileCode: Select all
def tmake(self,offset='.'):
print('\nMAKE')
print('> clean .log \n> copy json_file to .log')
# my_file_clear_folder(self.__log_path)
for root, dirs, files in os.walk(self.__log_path):
for name in files:
if name.endswith(".bin"):
os.remove(os.path.join(root, name))preacher 2022年 Sep 21日 14:07可以参照下面修改下my_ide_gcc.py和my_ide_base.py脚本，可以实现增量编译
- 在my_ide_gcc.py文件的my_ide_gcc类中增加def if_need_rebuild函数，并按照下面代码修改__compile函数
Code: Select all
def if_need_rebuild(self,dep_file): with open(dep_file,"r") as f: list_line1 = f.readline().split(' ') if(list_line1[1] != '\\\n'): try: if(os.path.getmtime(list_line1[1]) > os.path.getmtime(dep_file)): return True except: return True for line in f.readlines(): line = line.lstrip() line = line.strip('\n') list_n = line.split(' ') for file in list_n: if(file == '\\'): break try: if(os.path.getmtime(file) > os.path.getmtime(dep_file)): return True except: print("file not exit") return True return False def __compile(self,kind,file,out_path,evn): cc = self.cmd['gcc']['cc'] asm = self.cmd['gcc']['asm'] c_flags = self.cmd['gcc']['c_flags'] c_macros = self.cmd['gcc']['c_macros'] s_flags = self.cmd['gcc']['s_flags'] o_file = out_path+'/'+os.path.splitext(os.path.basename(file))[0]+'.o' d_file = o_file+'.d' if kind == '.c': gcc_h_file = out_path+'/'+'gcc_h.txt' Note=open(gcc_h_file,mode='w') Note.write(self.src['h_dir_str']) Note.close() if((os.path.exists(d_file) == True) and (self.if_need_rebuild(d_file) == False)): return o_file if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s @%s -c %s -o %s %s -MMD -MF %s"%(cc,c_flags,gcc_h_file,file,o_file,c_macros, d_file) elif kind == '.s': if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s -c %s -o %s"%(asm,s_flags,file,o_file) my_exe_simple(cmd,1,evn,self.var_map) return o_file
- 按照下面代码修改my_ide_base.py中的tmake函数
Code: Select all
def tmake(self,offset='.'): print('\nMAKE') print('> clean .log \n> copy json_file to .log') # my_file_clear_folder(self.__log_path) for root, dirs, files in os.walk(self.__log_path): for name in files: if name.endswith(".bin"): os.remove(os.path.join(root, name))
- 重新编译，第一次会全量编译，后面就会增量编译了。
好的，谢谢
preacher 2022年 Sep 21日 14:07可以参照下面修改下my_ide_gcc.py和my_ide_base.py脚本，可以实现增量编译
- 在my_ide_gcc.py文件的my_ide_gcc类中增加def if_need_rebuild函数，并按照下面代码修改__compile函数
Code: Select all
def if_need_rebuild(self,dep_file): with open(dep_file,"r") as f: list_line1 = f.readline().split(' ') if(list_line1[1] != '\\\n'): try: if(os.path.getmtime(list_line1[1]) > os.path.getmtime(dep_file)): return True except: return True for line in f.readlines(): line = line.lstrip() line = line.strip('\n') list_n = line.split(' ') for file in list_n: if(file == '\\'): break try: if(os.path.getmtime(file) > os.path.getmtime(dep_file)): return True except: print("file not exit") return True return False def __compile(self,kind,file,out_path,evn): cc = self.cmd['gcc']['cc'] asm = self.cmd['gcc']['asm'] c_flags = self.cmd['gcc']['c_flags'] c_macros = self.cmd['gcc']['c_macros'] s_flags = self.cmd['gcc']['s_flags'] o_file = out_path+'/'+os.path.splitext(os.path.basename(file))[0]+'.o' d_file = o_file+'.d' if kind == '.c': gcc_h_file = out_path+'/'+'gcc_h.txt' Note=open(gcc_h_file,mode='w') Note.write(self.src['h_dir_str']) Note.close() if((os.path.exists(d_file) == True) and (self.if_need_rebuild(d_file) == False)): return o_file if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s @%s -c %s -o %s %s -MMD -MF %s"%(cc,c_flags,gcc_h_file,file,o_file,c_macros, d_file) elif kind == '.s': if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s -c %s -o %s"%(asm,s_flags,file,o_file) my_exe_simple(cmd,1,evn,self.var_map) return o_file
- 按照下面代码修改my_ide_base.py中的tmake函数
Code: Select all
def tmake(self,offset='.'): print('\nMAKE') print('> clean .log \n> copy json_file to .log') # my_file_clear_folder(self.__log_path) for root, dirs, files in os.walk(self.__log_path): for name in files: if name.endswith(".bin"): os.remove(os.path.join(root, name))
- 重新编译，第一次会全量编译，后面就会增量编译了。
def tmake(self,offset='.'):
print('\nMAKE')
print('> clean .log \n> copy json_file to .log')
Code: Select all
# my_file_clear_folder(self.__log_path)
for root, dirs, files in os.walk(self.__log_path):
for name in files:
if name.endswith(".bin"):
os.remove(os.path.join(root, name))
my_file_copy_files_to([self.json_file],self.__log_path)
self.json_file = self.__log_path+'/'+os.path.basename(self.json_file)
my_file_str_replace(self.json_file,'$PROJECT_ROOT',offset)#PROJECT_PATH
with open(self.json_file,'r') as load_f:
load_dict = json.load(load_f)
print('#1. fill the output dict')
tool = load_dict['tool'][self.ide_kind]
self.output = load_dict['output']
self.output['sdk'].update({'components':load_dict['components']})
self.output['fw'].update({'output':tool['output']})
print('#2. fill the cmd dict')
self.cmd = {**tool['toolchain'], **tool['cmd']}
print('#3. fill flash cmd dict')
self.flash['bin_path'] = my_file_get_abs_path_and_formart(tool['flash']['bin_path'])
self.flash['flash_user_cmd'] = tool['flash']['flash_user_cmd']
self.flash['flash_all_cmd'] = tool['flash']['flash_all_cmd']
print('#4. fill the src dict')
self.__json_deep_search(load_dict)
# h_dirs list change to string
for h_dir in self.src['h_dirs']:
self.src['h_dir_str'] += (' -I'+h_dir)
# get l_dirs and change to string
# l_files change to string
l_dirs = []
for l_file in self.src['l_files']:
self.src['l_files_str'] += (' -l'+os.path.splitext(os.path.basename(l_file))[0][3:])
l_dir = os.path.dirname(l_file)
if l_dir not in l_dirs:
l_dirs.append(l_dir)
self.src['l_dirs_str'] += (' -L'+l_dir)
print('#5. get var map')
self.__get_variable_map()按照上面这样写，修改一个c文件还是会编译很多内容，但是效果有所改善，编译速度也快了很多
rockjablew 2022年 Sep 21日 17:48preacher 2022年 Sep 21日 14:07可以参照下面修改下my_ide_gcc.py和my_ide_base.py脚本，可以实现增量编译
- 在my_ide_gcc.py文件的my_ide_gcc类中增加def if_need_rebuild函数，并按照下面代码修改__compile函数
Code: Select all
def if_need_rebuild(self,dep_file): with open(dep_file,"r") as f: list_line1 = f.readline().split(' ') if(list_line1[1] != '\\\n'): try: if(os.path.getmtime(list_line1[1]) > os.path.getmtime(dep_file)): return True except: return True for line in f.readlines(): line = line.lstrip() line = line.strip('\n') list_n = line.split(' ') for file in list_n: if(file == '\\'): break try: if(os.path.getmtime(file) > os.path.getmtime(dep_file)): return True except: print("file not exit") return True return False def __compile(self,kind,file,out_path,evn): cc = self.cmd['gcc']['cc'] asm = self.cmd['gcc']['asm'] c_flags = self.cmd['gcc']['c_flags'] c_macros = self.cmd['gcc']['c_macros'] s_flags = self.cmd['gcc']['s_flags'] o_file = out_path+'/'+os.path.splitext(os.path.basename(file))[0]+'.o' d_file = o_file+'.d' if kind == '.c': gcc_h_file = out_path+'/'+'gcc_h.txt' Note=open(gcc_h_file,mode='w') Note.write(self.src['h_dir_str']) Note.close() if((os.path.exists(d_file) == True) and (self.if_need_rebuild(d_file) == False)): return o_file if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s @%s -c %s -o %s %s -MMD -MF %s"%(cc,c_flags,gcc_h_file,file,o_file,c_macros, d_file) elif kind == '.s': if (os.path.exists(o_file)): my_file_rm_file(o_file) cmd = "%s %s -c %s -o %s"%(asm,s_flags,file,o_file) my_exe_simple(cmd,1,evn,self.var_map) return o_file
- 按照下面代码修改my_ide_base.py中的tmake函数
Code: Select all
def tmake(self,offset='.'): print('\nMAKE') print('> clean .log \n> copy json_file to .log') # my_file_clear_folder(self.__log_path) for root, dirs, files in os.walk(self.__log_path): for name in files: if name.endswith(".bin"): os.remove(os.path.join(root, name))
- 重新编译，第一次会全量编译，后面就会增量编译了。
make
def tmake(self,offset='.'):
print('\nMAKE')
print('> clean .log \n> copy json_file to .log')Code: Select all
# my_file_clear_folder(self.__log_path) for root, dirs, files in os.walk(self.__log_path): for name in files: if name.endswith(".bin"): os.remove(os.path.join(root, name)) my_file_copy_files_to([self.json_file],self.__log_path) self.json_file = self.__log_path+'/'+os.path.basename(self.json_file) my_file_str_replace(self.json_file,'$PROJECT_ROOT',offset)#PROJECT_PATH with open(self.json_file,'r') as load_f: load_dict = json.load(load_f) print('#1. fill the output dict') tool = load_dict['tool'][self.ide_kind] self.output = load_dict['output'] self.output['sdk'].update({'components':load_dict['components']}) self.output['fw'].update({'output':tool['output']}) print('#2. fill the cmd dict') self.cmd = {**tool['toolchain'], **tool['cmd']} print('#3. fill flash cmd dict') self.flash['bin_path'] = my_file_get_abs_path_and_formart(tool['flash']['bin_path']) self.flash['flash_user_cmd'] = tool['flash']['flash_user_cmd'] self.flash['flash_all_cmd'] = tool['flash']['flash_all_cmd'] print('#4. fill the src dict') self.__json_deep_search(load_dict) # h_dirs list change to string for h_dir in self.src['h_dirs']: self.src['h_dir_str'] += (' -I'+h_dir) # get l_dirs and change to string # l_files change to string l_dirs = [] for l_file in self.src['l_files']: self.src['l_files_str'] += (' -l'+os.path.splitext(os.path.basename(l_file))[0][3:]) l_dir = os.path.dirname(l_file) if l_dir not in l_dirs: l_dirs.append(l_dir) self.src['l_dirs_str'] += (' -L'+l_dir) print('#5. get var map') self.__get_variable_map()按照上面这样写，修改一个c文件还是会编译很多内容，但是效果有所改善，编译速度也快了很多
我们争取本周加上俩功能：
1）差量编译（也会提升很多）
2）多线程编译（比makefile的多核编译还要快很多，就是您上面体验到的效果）
到时候您只要删除.ide_tool，然后编译时 vscode会自动拉取，就能体验最新的功能了