#!/usr/bin/env python# -*- Coding: utf-8 -*-## copyright 2019 The FATE Authors. All Rights Reserved.## licensed under the Apache license,Version 2.0 (the "license");# you may not use this file except in compliance with the license.# You may obtain a copy of the license at## http://www.apache.org/licenses/liCENSE-2.0## Unless required by applicable law or agreed to in writing,software# distributed under the license is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implIEd.# See the license for the specific language governing permissions and# limitations under the license.import Jsonimport osimport subprocessimport sysimport time# from load_file_functions import load_filehome_dir = os.path.split(os.path.realpath(__file__))[0]config_path = home_dir + ‘/conf‘data_path = home_dir + ‘/../data‘load_file_program = home_dir + ‘/../load_file/load_file.py‘data_set = ‘breast‘# data_set = ‘default_credit‘# data_set = ‘give_credit‘# data_set = ‘breast_onehot‘# data_set = ‘student-mat‘mode = ‘fit‘# mode = ‘transform‘def make_config_file(work_mode,job_ID,role,guest_partyID,host_partyID): work_mode = int(work_mode) with open(config_path + ‘/{}_runtime_conf.Json‘.format(role),‘r‘,enCoding=‘utf-8‘) as load_f: role_config = Json.load(load_f) if role == ‘guest‘: role_config[‘local‘][‘party_ID‘] = guest_partyID data_suffix = ‘b‘ else: role_config[‘local‘][‘party_ID‘] = host_partyID data_suffix = ‘a‘ role_config[‘role‘][‘host‘][0] = host_partyID role_config[‘role‘][‘guest‘][0] = guest_partyID role_config[‘WorkFlowParam‘][‘work_mode‘] = int(work_mode) role_config[‘FeatureBinningParam‘][‘process_method‘] = mode role_config[‘WorkFlowParam‘][‘train_input_table‘] = "{}_{}_{}".format(data_set,job_ID) # guest_config_path = config_path + ‘/guest_runtime_conf.Json_‘ + str(job_ID) role_config_path = "{}/{}_runtime_conf.Json_{}".format( config_path,job_ID ) with open(role_config_path,‘w‘,enCoding=‘utf-8‘) as Json_file: Json.dump(role_config,Json_file,ensure_ascii=False) with open(config_path + ‘/load_file.Json‘,enCoding=‘utf-8‘) as load_f: load_config = Json.load(load_f) load_config[‘work_mode‘] = work_mode # load_config[‘file‘] = data_path + ‘/‘ + data_set + ‘_b.csv‘ load_config[‘file‘] = "{}/{}_{}.csv".format(data_path,data_set,data_suffix) load_config[‘table_name‘] = "{}_{}_{}".format(data_set,job_ID) # load_file_role = config_path + ‘/load_file.Json_guest_‘ + str(job_ID) load_file_path = "{}/load_file.Json_{}_{}".format(config_path,job_ID) with open(load_file_path,enCoding=‘utf-8‘) as Json_file: Json.dump(load_config,ensure_ascii=False) return role_config_path,load_file_pathdef load_file(load_file_path): load_process = subprocess.Popen(["python",load_file_program,"-c",load_file_path,]) # load_process.communicate() returncode = load_process.wait() print("Load file return code : {}".format(returncode))def attach_all_result_log(running_process,role): while True: poll = running_process.poll() if poll is None: print(‘Workflow is still running‘) time.sleep(5) else: break log_dir = home_dir + ‘/../../logs/{}‘.format(job_ID) role = role.lower() if role == ‘guest‘: role_List = [‘Guest‘,‘guest‘,‘GUEST‘] else: role_List = [‘Host‘,‘host‘,‘HOST‘] result_log_List = [] for log_file in os.Listdir(log_dir): # print(‘current log_file: {}‘.format(log_file)) with open(log_dir + ‘/‘ + log_file,enCoding=‘utf-8‘) as f: for log_line in f: if ‘Result‘ in log_line: for r in role_List: if r in log_line: result_log_List.append(log_line) break log_List = sorted(result_log_List,key=get_time) for log_line in log_List: print(log_line)def get_time(log_line): time_str = log_line.split(‘ - ‘)[0] time_str = time_str[1:] time_str.strip() time_sec,time_mili_sec = time_str.split(‘,‘) timestamp = time.mktime(time.strptime(time_sec,‘%Y-%m-%d %H:%M:%s‘)) timestamp += float(time_mili_sec) / 1000 return timestampif __name__ == ‘__main__‘: work_mode = sys.argv[1] jobID = sys.argv[2] role = sys.argv[3] guest_partyID = int(sys.argv[4]) host_partyID = int(sys.argv[5]) role_config_path,load_file_path = make_config_file(work_mode,jobID,host_partyID) load_file(load_file_path) work_path = home_dir + ‘/../../workflow/hetero_binning_workflow/‘ ‘hetero_binning_{}_workflow.py‘.format(role) workflow_process = subprocess.Popen(["python",work_path,role_config_path,"-j",jobID ]) returncode = workflow_process.wait() print("Load file return code : {}".format(returncode)) attach_all_result_log(workflow_process,role)总结
以上是内存溢出为你收集整理的featurebining全部内容,希望文章能够帮你解决featurebining所遇到的程序开发问题。
如果觉得内存溢出网站内容还不错,欢迎将内存溢出网站推荐给程序员好友。
欢迎分享,转载请注明来源:内存溢出
评论列表(0条)