使用GenAI做Discord舆情分析对游戏运营的帮助

sudo yum install https://rpm.nodesource.com/pub_18.x/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y 
sudo yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1 --nogpgcheck

sudo yum install docker -y
sudo service docker start
sudo chmod 666 /var/run/docker.sock

sudo yum install git -y
sudo yum install jq

sudo npm install -g aws-cdk
sudo npm install --global yarn

wget https://www.python.org/ftp/python/3.11.8/Python-3.11.8.tgz 
tar xzf Python-3.11.8.tgz 
cd Python-3.11.8 
sudo ./configure --enable-optimizations 

CDK 2.122.0+
nodejs 18+
npm 10+
python 3.11+
git

git clone https://github.com/aws-samples/discord-community-insights-with-GenAI.git
cd llm-text-keyword-summary/deploy

#!/bin/bash
account_id=`aws sts get-caller-identity --query "Account" --output text`
ts=`date +%y-%m-%d-%H-%M-%S`
unique_tag="$account_id-$ts"
# Glue 作业名称
GLUE_JOB_NAME="llm-analysis-text-job"
# Glue Discord 作业名称
GLUE_DISCORD_JOB_NAME="discord-message-collect-job"
# Glue Discord 作业名称
GLUE_SUMMARIZE_JOB_NAME="llm-summarize-job"
# 存储discord token secretname
DISCORD_SECRET_NAME="discord-token"
# DynamoDB中存储prompt的表名
LLM_ANALYSIS_TEXT_TABLE_NAME="prompt-template"
# 存储原始数据以及分析结果的S3名称
S3_BUCKET_NAME="llm-analysis-text-${unique_tag}"
# S3中原始数据Prefix的名称
RAW_DATA_PREFIX="raw-data/"
# Glue Catalog中DB名称
GLUE_DATABASE="llm_text_db"
# Glue Catalog中查询S3中分析数据使用表名
GLUE_TABLE="sentiment_result"
# Glue Catalog中查询S3中总结数据使用表名
GLUE_SUMMARY_TABLE="summary_result"
echo "GLUE_JOB_NAME=${GLUE_JOB_NAME}" > .env
echo "GLUE_DISCORD_JOB_NAME=${GLUE_DISCORD_JOB_NAME}" >> .env
echo "GLUE_SUMMARIZE_JOB_NAME=${GLUE_SUMMARIZE_JOB_NAME}" >> .env
echo "LLM_ANALYSIS_TEXT_TABLE_NAME=${LLM_ANALYSIS_TEXT_TABLE_NAME}" >> .env
echo "S3_BUCKET_NAME=${S3_BUCKET_NAME}" >> .env
echo "RAW_DATA_PREFIX=${RAW_DATA_PREFIX}" >> .env
echo "GLUE_DATABASE=${GLUE_DATABASE}" >> .env
echo "GLUE_TABLE=${GLUE_TABLE}" >> .env
echo "GLUE_SUMMARY_TABLE=${GLUE_SUMMARY_TABLE}" >> .env
echo "DISCORD_SECRET_NAME=${DISCORD_SECRET_NAME}" >> .env

bash ./gen_env.sh 

npm install
cdk bootstrap
cdk synth
cdk deploy

CREATE EXTERNAL TABLE sentiment_result (
    chat STRING,
    sentiment STRING
)
PARTITIONED BY (
  job_id STRING
)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
WITH SERDEPROPERTIES ('ignore.malformed.json' = 'true')
LOCATION "s3://<替换上面配置的S3 Bucket名称>/result/"
TBLPROPERTIES (
  "projection.enabled" = "true",
  "projection.job_id.type" = "injected",
  "storage.location.template" = "s3://<替换上面配置的S3 Bucket名称>/result/job_id=${job_id}"
)

CREATE EXTERNAL TABLE summary_result (
    counts STRING,
    summary STRING
)
PARTITIONED BY (
  job_id STRING
)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
WITH SERDEPROPERTIES ('ignore.malformed.json' = 'true')
LOCATION "s3://<替换上面配置的S3 Bucket名称>/summary/"
TBLPROPERTIES (
  "projection.enabled" = "true",
  "projection.job_id.type" = "injected",
  "storage.location.template" = "s3://<替换上面配置的S3 Bucket名称>/summary/job_id=${job_id}"
)

CREATE EXTERNAL TABLE user_jobs (
    channel_id STRING,
    channel_name STRING,
    message_count INT,
    job_run_id STRING,
    run_time STRING,
    timestamp BIGINT
)
PARTITIONED BY (
  username STRING
)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
WITH SERDEPROPERTIES ('ignore.malformed.json' = 'true')
LOCATION "s3://<替换上面配置的S3 Bucket名称>/user-jobs/"
TBLPROPERTIES (
  "projection.enabled" = "true",
  "projection.username.type" = "injected",
  "storage.location.template" = "s3://<替换上面配置的S3 Bucket名称>/user-jobs/username=${username}"
)

def get_discord_token():
    secret_name = "discord-token"

    # Create a Secrets Manager client
    session = boto3.session.Session()
    client = session.client(
        service_name='secretsmanager'
    )

    try:
        get_secret_value_response = client.get_secret_value(
            SecretId=secret_name
        )
    except ClientError as e:
        raise e

    return json.loads(get_secret_value_response['SecretString'])

async def get_recent_messages(channel):
    print("Calculating one_week_ago...")
    data_period = int(token_info.get('DATA_PERIOD'))
    one_week_ago = datetime.now() - timedelta(days=data_period)
    print(f"one_week_ago: {one_week_ago}")
    messages = [message async for message in channel.history(after=one_week_ago)]
    message_data = [to_json(message) for message in messages]
    print(f"Messages count: {len(message_data)}")
    return message_data

prompt_sentiment = '''
You are a chat message sentiment classifer
Here is a document you will classify the senetiment
<doc>
{content}
</doc>
please list all the content then classify the sentiment of each content into [positive,neutral,negative]'
Please follow below requirements:
1. You will strictly be based on the document in <doc>.
2. please enclose your analysis results in xml tag <sentiment>.
for example:
<sentiment>
1. "auction is very good." [positive]
2. "I got a good deal at the auction." [positive]
3. " The auction house was terrible." [negative]
4. "auction sucks" [negative]
5. "There were people giving out red envelopes (with money) at the auction house." [neutral]
</sentiment>
Skip the preamble, go straight into the answer.
'''
print('prompt_sentiment:',prompt_sentiment)

class CustOuputParser(BaseOutputParser[str]):

    def extract(self, content: str) -> tuple[str, str]:
        pattern = r'"(.*?)" \[(.*?)\]'
        matches = re.findall(pattern, content)
        if matches:
            return [(text, sentiment) for text, sentiment in matches]

    def parse(self, text: str) -> str:
        results = self.extract(text)
        output = []
        for text, sentiment in results:
            output.append(json.dumps({"chat": text, "sentiment": sentiment}, ensure_ascii=False))
        return "\n".join(output)

    @property
    def _type(self) -> str:
        return "cust_output_parser"

prompt_sentiment = ChatPromptTemplate.from_template(prompt_sentiment)
llm_sonnet = ChatBedrock(model_id="anthropic.claude-3-sonnet-20240229-v1:0",
                  model_kwargs={"temperature": 0,
                                "top_k":10,
                                "max_tokens": 1024,
                                "top_p":0.5,
                                # "stop_sequences":['</response>']
                               })
output_parser  = CustOuputParser()
sentiment_chain = prompt_sentiment | llm_sonnet | output_parser

df = wr.athena.read_sql_query(f'SELECT * FROM sentiment_result where job_id=\'{job_run_id}\'', database=glue_db)
print(df)
sentiment_counts = df.groupby('sentiment').size()
print(sentiment_counts)
sentiment_counts_json = sentiment_counts.to_json(orient='index')
print(sentiment_counts_json)
                              
chat_content = df['chat']
# 使用str.cat()方法将每一行内容添加回车符号并连接在一起
combined_chat = chat_content.str.cat(sep='\n')
print(combined_chat)
text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
    chunk_size=1000, chunk_overlap=100
)
split_docs = text_splitter.split_text(combined_chat)
docs = [Document(page_content=doc) for doc in split_docs]
llm_summarize = ChatBedrock(model_id="anthropic.claude-3-sonnet-20240229-v1:0",
                  model_kwargs={"temperature": 0,
                                "top_k":10,
                                "max_tokens": 1024,
                                "top_p":0.5
                               })

prompt_template = """You are the summarizer of chat records for game players. You need to summarize the chat records within <content> in Chinese, focusing primarily on opinions regarding in-game mechanisms and game items.
You need to provide a summary based on three dimensions: positive feedback, negative feedback, and player suggestions. Additionally, you should provide typical player chat for each dimension.
<content>
{text}
</content>
CONCISE SUMMARY:"""
prompt = PromptTemplate.from_template(prompt_template)
refine_template = (
    "You are the summarizer of chat records for game players, your job is to produce a final summary in Chinese\n"
    "We have provided an existing summary up to a certain point: {existing_answer}\n"
    "We have the opportunity to refine the existing summary"
    "(only if needed) with some more context below.\n"
    "------------\n"
    "{text}\n"
    "------------\n"
    "Given the new context, refine the original summary"
    "If the context isn't useful, return the original summary."
)

cd llm-text-keyword-summary/demo
pip3 install -r ./requirements.txt

#!/bin/bash
# Api Gateway 暴露的domain URL，需要在CDK 部署完毕后进行配置
domain_url="https://xxxx.execute-api.us-east-1.amazonaws.com/prod/"
# 调用Api Gateway 暴露的domain URL API key，需要在CDK 部署完毕后进行配置
apikeys=""
echo "domain_url=${domain_url}" >> .env
echo "apikeys=${apikeys}" >> .env

bash ./gen_demo_env.sh

vi config.yaml

    <username1>:
      email: <username1@amazon.com>
      failed_login_attempts: 0 # Will be managed automatically
      logged_in: True # Will be managed automatically
      name: <User Test>
      password: < >

python3 password_generator.py <password>

streamlit run demo_app.py --server.port 6001