Python中extract_tags()怎么对多行文本提取特征词而不是一行一行...

发布网友发布时间：2024-03-26 15:34

共1个回答

热心网友时间：2024-07-31 05:12

[python] view plain copy
#coding:utf-8
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from multiprocessing import Pool,Queue,Process
import multiprocessing as mp
import time,random
import os
import codecs
import jieba.analyse
jieba.analyse.set_stop_words("yy_stop_words.txt")

def extract_keyword(input_string):
#print("Do task by process {proc}".format(proc=os.getpid()))
tags = jieba.analyse.extract_tags(input_string, topK=100)
#print("key words:{kw}".format(kw=" ".join(tags)))
return tags

#def parallel_extract_keyword(input_string,out_file):
def parallel_extract_keyword(input_string):
#print("Do task by process {proc}".format(proc=os.getpid()))
tags = jieba.analyse.extract_tags(input_string, topK=100)
#time.sleep(random.random())
#print("key words:{kw}".format(kw=" ".join(tags)))
#o_f = open(out_file,'w')
#o_f.write(" ".join(tags)+"\n")
return tags
if __name__ == "__main__":

data_file = sys.argv[1]
with codecs.open(data_file) as f:
lines = f.readlines()
f.close()

out_put = data_file.split('.')[0] +"_tags.txt"
t0 = time.time()
for line in lines:
parallel_extract_keyword(line)
#parallel_extract_keyword(line,out_put)
#extract_keyword(line)
print("串行处理花费时间{t}".format(t=time.time()-t0))

pool = Pool(processes=int(mp.cpu_count()*0.7))
t1 = time.time()
#for line in lines:
#pool.apply_async(parallel_extract_keyword,(line,out_put))
#保存处理的结果，可以方便输出到文件
res = pool.map(parallel_extract_keyword,lines)
#print("Print keywords:")
#for tag in res:
#print(" ".join(tag))

pool.close()
pool.join()
print("并行处理花费时间{t}s".format(t=time.time()-t1))

运行：
python data_process_by_multiprocess.py message.txt
message.txt是每行是一个文档，共581行，7M的数据

运行时间：

不使用sleep来挂起进程,也就是把time.sleep(random.random())注释掉，运行可以大大节省时间。