首页   注册   登录
V2EX = way to explore
V2EX 是一个关于分享和探索的地方
现在注册
已注册用户请  登录
V2EX  ›  allenwuli  ›  全部回复第 1 页 / 共 2 页
回复总数  33
1  2  
???
我之前也碰到,我用的 django。业务场景跟你差不多,我后来用了类似 8 楼的方法。我单独开了个进程去查询数据库,没有想到其他的方法。
@shyrock 你要没抢到,又没说不给你退。
整点抢,前一百名 618。我也是在天猫官方微博上看到的: https://weibointl.api.weibo.cn/share/76713577.html?weibo_id=4384522026270170
分母
187 天前
回复了 allenwuli 创建的主题 iPhone PDD 上的 iPhone 为何那么便宜?
@l33ch 这价格降的太快,真想剁手啊
191 天前
回复了 allenwuli 创建的主题 算法 超大数据量排序算法
import os
import argparse


class FileSplitter(object):
BLOCK_FILENAME_FORMAT = 'block_{0}.dat'

def __init__(self, filename):
self.filename = filename
self.block_filenames = []

def write_block(self, data, block_number):
filename = self.BLOCK_FILENAME_FORMAT.format(block_number)
with open(filename, 'w') as file:
file.write(data)
self.block_filenames.append(filename)

def get_block_filenames(self):
return self.block_filenames

def split(self, block_size, sort_key=None):
i = 0
with open(self.filename) as file:
while True:
lines = file.readlines(block_size)

if not lines:
break

if sort_key is None:

lin = [int(aaa.split("\n")[0]) for aaa in lines]
lin.sort()
lines = [str(sss) + "\n" for sss in lin]
else:
lin = [int(aaa.split("\n")[0]) for aaa in lines]
lin.sort(key=sort_key)
lines = [str(sss) + "\n" for sss in lin]

self.write_block(''.join(lines), i)
i += 1

def cleanup(self):
map(lambda f: os.remove(f), self.block_filenames)


class NWayMerge(object):
def select(self, choices):
print(choices)
min_index = -1
min_str = None
for i in choices.keys():
if min_str is None or choices[i] < min_str:
min_index = i
min_str = choices[i]
print(min_index)
return min_index


class FilesArray(object):
def __init__(self, files):
self.files = files
self.empty = set()
self.num_buffers = len(files)
self.buffers = {i: None for i in range(self.num_buffers)}

def get_dict(self):
return {i: int(self.buffers[i].split("\n")[0]) for i in range(self.num_buffers) if i not in self.empty}

def refresh(self):
for i in range(self.num_buffers):
if self.buffers[i] is None and i not in self.empty:
self.buffers[i] = self.files[i].readline()
if self.buffers[i] == '':
self.empty.add(i)
self.files[i].close()
if len(self.empty) == self.num_buffers:
return False

return True

def unshift(self, index):
value = self.buffers[index]
self.buffers[index] = None

return value


class FileMerger(object):
def __init__(self, merge_strategy):
self.merge_strategy = merge_strategy

def merge(self, filenames, outfilename, buffer_size):
buffers = FilesArray(self.get_file_handles(filenames, buffer_size))
with open(outfilename, 'w', buffer_size) as outfile:
while buffers.refresh():
min_index = self.merge_strategy.select(buffers.get_dict())
outfile.write(buffers.unshift(min_index))

def get_file_handles(self, filenames, buffer_size):
files = {}

for i in range(len(filenames)):
files[i] = open(filenames[i], 'r', buffer_size)

return files


class ExternalSort(object):
def __init__(self, block_size):
self.block_size = block_size

def sort(self, filename, sort_key=None):
num_blocks = self.get_number_blocks(filename, self.block_size)
splitter = FileSplitter(filename)
splitter.split(self.block_size, sort_key)

merger = FileMerger(NWayMerge())
buffer_size = self.block_size / (num_blocks + 1)
merger.merge(splitter.get_block_filenames(), filename + '.out', int(buffer_size))

splitter.cleanup()

def get_number_blocks(self, filename, block_size):
return (os.stat(filename).st_size / block_size) + 1


def parse_memory(string):
if string[-1].lower() == 'k':
return int(string[:-1]) * 1024
elif string[-1].lower() == 'm':
return int(string[:-1]) * 1024 * 1024
elif string[-1].lower() == 'g':
return int(string[:-1]) * 1024 * 1024 * 1024
else:
return int(string)


def main():

# parser = argparse.ArgumentParser()
# parser.add_argument('-m',
# '--mem',
# help='amount of memory to use for sorting',
# default='100M')
# parser.add_argument('filename',
# metavar='<filename>',
# nargs=1,
# help='name of file to sort')
# args = parser.parse_args()

sorter = ExternalSort(300)
sorter.sort('num1.txt')


if __name__ == '__main__':
main()
195 天前
回复了 allenwuli 创建的主题 算法 超大数据量排序算法
@jmc891205 谢了老板
206 天前
回复了 allenwuli 创建的主题 macOS macOS Mojave 10.14.5(18F132)卡顿
@cocoking 没有,使用 chrome 刷网页的时候,还是会不时的卡一下。
206 天前
回复了 allenwuli 创建的主题 macOS macOS Mojave 10.14.5(18F132)卡顿
@cdlixucd 不知道啊,早知道不升级了。
1  2  
关于   ·   FAQ   ·   API   ·   我们的愿景   ·   广告投放   ·   感谢   ·   实用小工具   ·   4401 人在线   最高记录 5043   ·     Select Language
创意工作者们的社区
World is powered by solitude
VERSION: 3.9.8.3 · 45ms · UTC 03:03 · PVG 11:03 · LAX 19:03 · JFK 22:03
♥ Do have faith in what you're doing.