当大火的文图生成模型遇见知识图谱，AI画像趋近于真实世界

# in easynlp/appzoo/text2image_generation/model.py# initself.transformer = GPT_knowl(self.config)# forwardx = inputs['image']c = inputs['text']words_emb = inputs['words_emb']x = x.permute(0, 3, 1, 2).to(memory_format=torch.contiguous_format)# one step to produce the logits_, z_indices = self.encode_to_z(x) c_indices = ccz_indices = torch.cat((c_indices, a_indices), dim=1)# make the predictionlogits, _ = self.transformer(cz_indices[:, :-1], words_emb, flag=True)# cut off conditioning outputs - output i corresponds to p(z_i | z_{<i}, c)logits = logits[:, c_indices.shape[1]-1:]

# in easynlp/appzoo/text2image_generation/data.py# preprocess word_matrixwords_mat = np.zeros([self.entity_num, self.text_len], dtype=np.int)if len(lex_id) > 0:    ents = lex_id.split(' ')[:self.entity_num]    pos_s = [int(x) for x in pos_s.split(' ')]    pos_e = [int(x) for x in pos_e.split(' ')]    ent_pos_s = pos_s[token_len:token_len+self.entity_num]    ent_pos_e = pos_e[token_len:token_len+self.entity_num]    for i, ent in enumerate(ents):        words_mat[i, ent_pos_s[i]:ent_pos_e[i]+1] = entencoding['words_mat'] = words_mat# in batch_fnwords_mat = torch.LongTensor([example['words_mat'] for example in batch])words_emb = self.embed(words_mat)

https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/artist_text2image/T2I_train.tsvhttps://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/artist_text2image/T2I_val.tsvhttps://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/artist_text2image/T2I_test.tsv

# 下载entity_id与entity_vector的映射表wget -P ./tmp https://atp-modelzoo-sh.oss-cn-shanghai.aliyuncs.com/release/tutorials/artist_text2image/entity2vec.pt# finetunepython -m torch.distributed.launch $DISTRIBUTED_ARGS examples/text2image_generation/main_knowl.py \    --mode=train \    --worker_gpu=1 \    --tables=./tmp/T2I_knowl_train.tsv,./tmp/T2I_knowl_val.tsv \    --input_schema=idx:str:1,text:str:1,lex_id:str:1,pos_s:str:1,pos_e:str:1,token_len:str:1,imgbase64:str:1,  \    --first_sequence=text \    --second_sequence=imgbase64 \    --checkpoint_dir=./tmp/artist_model_finetune \    --learning_rate=4e-5 \    --epoch_num=2 \    --random_seed=42 \    --logging_steps=100 \    --save_checkpoint_steps=200 \    --sequence_length=288 \    --micro_batch_size=8 \    --app_name=text2image_generation \    --user_defined_parameters='        pretrain_model_name_or_path=alibaba-pai/pai-artist-knowl-base-zh        entity_emb_path=./tmp/entity2vec.pt        size=256        text_len=32        img_len=256        img_vocab_size=16384      ' # predictpython -m torch.distributed.launch $DISTRIBUTED_ARGS examples/text2image_generation/main_knowl.py \    --mode=predict \    --worker_gpu=1 \    --tables=./tmp/T2I_knowl_test.tsv \    --input_schema=idx:str:1,text:str:1,lex_id:str:1,pos_s:str:1,pos_e:str:1,token_len:str:1, \    --first_sequence=text \    --outputs=./tmp/T2I_outputs_knowl.tsv \    --output_schema=idx,text,gen_imgbase64 \    --checkpoint_dir=./tmp/artist_model_finetune \    --sequence_length=288 \    --micro_batch_size=8 \    --app_name=text2image_generation \    --user_defined_parameters='        entity_emb_path=./tmp/entity2vec.pt        size=256        text_len=32        img_len=256        img_vocab_size=16384        max_generated_num=4      '