Added some moderation features

code/cogs/moderation.py

@@ -13,18 +13,62 @@ class Moderation (discord.Cog):
     @discord.option(name="enable", description="Enable or disable AI moderation", reqired=True,)
     @discord.option(name="log_channel", description="The channel where the moderation logs will be sent", required=True)
     @discord.option(name="moderator_role", description="The role of the moderators", required=True)
+    #the types of toxicity are     'requestedAttributes': {'TOXICITY': {}, 'SEVERE_TOXICITY': {}, 'IDENTITY_ATTACK': {}, 'INSULT': {}, 'PROFANITY': {}, 'THREAT': {}, 'SEXUALLY_EXPLICIT': {}, 'FLIRTATION': {}, 'OBSCENE': {}, 'SPAM': {}},
+    @discord.option(name="toxicity", description="The toxicity threshold", required=False)
+    @discord.option(name="severe_toxicity", description="The severe toxicity threshold", required=False)
+    @discord.option(name="identity_attack", description="The identity attack threshold", required=False)
+    @discord.option(name="insult", description="The insult threshold", required=False)
+    @discord.option(name="profanity", description="The profanity threshold", required=False)
+    @discord.option(name="threat", description="The threat threshold", required=False)
+    @discord.option(name="sexually_explicit", description="The sexually explicit threshold", required=False)
+    @discord.option(name="flirtation", description="The flirtation threshold", required=False)
+    @discord.option(name="obscene", description="The obscene threshold", required=False)
+    @discord.option(name="spam", description="The spam threshold", required=False)
     #we set the default permissions to the administrator permission, so only the server administrators can use this command
     @default_permissions(administrator=True)
-    async def moderation(self, ctx: discord.ApplicationContext, enable: bool, log_channel: discord.TextChannel, moderator_role: discord.Role):
+    async def moderation(self, ctx: discord.ApplicationContext, enable: bool, log_channel: discord.TextChannel, moderator_role: discord.Role, toxicity: float = None, severe_toxicity: float = None, identity_attack: float = None, insult: float = None, profanity: float = None, threat: float = None, sexually_explicit: float = None, flirtation: float = None, obscene: float = None, spam: float = None):
         try: 
             data = c.execute("SELECT * FROM moderation WHERE guild_id = ?", (str(ctx.guild.id),))
             data = c.fetchone()
         except: data = None
         if data is None:
-            c.execute("INSERT INTO moderation VALUES (?, ?, ?, ?)", (str(ctx.guild.id), str(log_channel.id), enable, str(moderator_role.id)))
+            #first we check if any of the values is none. If it's none, we set it to 0.40
+            if toxicity is None: toxicity = 0.40
+            if severe_toxicity is None: severe_toxicity = 0.40
+            if identity_attack is None: identity_attack = 0.40
+            if insult is None: insult = 0.40
+            if profanity is None: profanity = 0.40
+            if threat is None: threat = 0.40
+            if sexually_explicit is None: sexually_explicit = 0.40
+            if flirtation is None: flirtation = 0.40
+            if obscene is None: obscene = 0.40
+            if spam is None: spam = 0.40
+            c.execute("INSERT INTO moderation VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (str(ctx.guild.id), str(log_channel.id), enable, str(moderator_role.id), toxicity, severe_toxicity, identity_attack, insult, profanity, threat, sexually_explicit, flirtation, obscene, spam))
             conn.commit()
+            await ctx.respond(content="Moderation has been enabled!", ephemeral=True)
         else:
-            c.execute("UPDATE moderation SET logs_channel_id = ?, is_enabled = ? WHERE guild_id = ?", (str(log_channel.id), enable, str(ctx.guild.id)))
+            #for each value we check if it's none. If it's none and there's no value in the database, we set it to 0.40, otherwise we set it to the value in the database
+            if toxicity is None and data[4] is not None: toxicity = data[4]
+            elif toxicity is None and data[4] is None: toxicity = 0.40
+            if severe_toxicity is None and data[5] is not None: severe_toxicity = data[5]
+            elif severe_toxicity is None and data[5] is None: severe_toxicity = 0.40
+            if identity_attack is None and data[6] is not None: identity_attack = data[6]
+            elif identity_attack is None and data[6] is None: identity_attack = 0.40
+            if insult is None and data[7] is not None: insult = data[7]
+            elif insult is None and data[7] is None: insult = 0.40
+            if profanity is None and data[8] is not None: profanity = data[8]
+            elif profanity is None and data[8] is None: profanity = 0.40
+            if threat is None and data[9] is not None: threat = data[9]
+            elif threat is None and data[9] is None: threat = 0.40
+            if sexually_explicit is None and data[10] is not None: sexually_explicit = data[10]
+            elif sexually_explicit is None and data[10] is None: sexually_explicit = 0.40
+            if flirtation is None and data[11] is not None: flirtation = data[11]
+            elif flirtation is None and data[11] is None: flirtation = 0.40
+            if obscene is None and data[12] is not None: obscene = data[12]
+            elif obscene is None and data[12] is None: obscene = 0.40
+            if spam is None and data[13] is not None: spam = data[13]
+            elif spam is None and data[13] is None: spam = 0.40
+            c.execute("UPDATE moderation SET logs_channel_id = ?, is_enabled = ?, mod_role_id = ?, toxicity = ?, severe_toxicity = ?, identity_attack = ?, insult = ?, profanity = ?, threat = ?, sexually_explicit = ?, flirtation = ?, obscene = ?, spam = ? WHERE guild_id = ?", (str(log_channel.id), enable, str(moderator_role.id), toxicity, severe_toxicity, identity_attack, insult, profanity, threat, sexually_explicit, flirtation, obscene, spam, str(ctx.guild.id)))
             conn.commit()
         await ctx.respond("Successfully updated moderation settings for this server", ephemeral=True)
 
@@ -44,28 +88,58 @@ class Moderation (discord.Cog):
         if not is_enabled: return
         content = message.content
         message_toxicity = tox.get_toxicity(content)
-        if message_toxicity >= 0.40:
+        reasons_to_delete = []
+        reasons_to_suspicous = []
+        for i in message_toxicity: 
+            if i >= float(data[message_toxicity.index(i)+4]): reasons_to_delete.append(tox.toxicity_names[message_toxicity.index(i)])
+        for i in message_toxicity:
+            if float(data[message_toxicity.index(i)+4]-0.1) <= i < float(data[message_toxicity.index(i)+4]): reasons_to_suspicous.append(tox.toxicity_names[message_toxicity.index(i)])
+        if len(reasons_to_delete) > 0:
+            embed = discord.Embed(title="Message deleted", description=f"Your message was deleted because it was too toxic. The following reasons were found: **{'**, **'.join(reasons_to_delete)}**", color=discord.Color.red())
+            await message.reply(f"{message.author.mention}", embed=embed, delete_after=15)
             await message.delete()
-            embed = discord.Embed(title="Message deleted", description=f"{message.author.mention} Your message was deleted because it was too toxic. Please keep this server safe and friendly. If you think this was a mistake, please contact a moderator.", color=discord.Color.red())
+            embed = discord.Embed(title="Message deleted", description=f"**{message.author}**'s message ***{content}*** was deleted because it was too toxic. The following reasons were found:", color=discord.Color.red())
-            await message.channel.send(f"{message.author.mention}", embed=embed, delete_after=15)
+            for i in reasons_to_delete:
-            formatted_message_sent_date = message.created_at.strftime("%d/%m/%Y %H:%M:%S")
+                toxicity_value = message_toxicity[tox.toxicity_names.index(i)]
-            embed = discord.Embed(title="Message deleted", description=f"The message \n***{content}***\n of {message.author.mention} sent in {message.channel.mention} on date **{formatted_message_sent_date}** was deleted because it was too toxic. The toxicity score was of **{message_toxicity*100}%**", color=discord.Color.red())
+                embed.add_field(name=i, value=f"Found toxicity value: **{toxicity_value*100}%**", inline=False)
             await channel.send(embed=embed)
-        elif 0.37 < message_toxicity < 0.40: #if the message is not toxic, but it is close to being toxic, we send a warning
+        elif len(reasons_to_suspicous) > 0:
-            embed = discord.Embed(title="Possible toxic message", description=f"A possible [toxic message: **{content}**]({message.jump_url}) was sent by {message.author.mention} in {message.channel.mention}. Please check it out.", color=discord.Color.orange())
+            await message.reply(f"{moderator_role.mention} This message might be toxic. The following reasons were found: **{'**, **'.join(reasons_to_suspicous)}**", delete_after=15, mention_author=False)
+            embed = discord.Embed(title="Message suspicious", description=f"**{message.author}**'s message [***{content}***]({message.jump_url}) might be toxic. The following reasons were found:", color=discord.Color.orange())
+            for i in reasons_to_suspicous:
+                toxicity_value = message_toxicity[tox.toxicity_names.index(i)]
+                embed.add_field(name=i, value=f"Found toxicity value: **{toxicity_value*100}%**", inline=False)
             await channel.send(embed=embed)
-            #we also reac with an orange circle emoji to the message
+            #we add a reaction to the message so the moderators can easily find it orange circle emoji
             await message.add_reaction("🟠")
-            #we reply to the message with a ping to the moderators
-            moderator_role = message.guild.get_role(int(data[3]))
-            await message.reply(f"Hey {moderator_role.mention}, this message might be toxic. Please check it out.", mention_author=False, delete_after=15)
-        else:
-            #the message is not toxic, so we don't do anything
-            pass
 
     @discord.slash_command(name="get_toxicity", description="Get the toxicity of a message")
     @discord.option(name="message", description="The message you want to check", required=True)
     @default_permissions(administrator=True)
     async def get_toxicity(self, ctx: discord.ApplicationContext, message: str):
-        toxicity = tox.get_toxicity(message)
+        response = tox.get_toxicity(message)
-        await ctx.respond(f"The toxicity of the message **{message}** is **{toxicity*100}%**")
+#        try: toxicity, severe_toxicity, identity_attack, insult, profanity, threat, sexually_explicit, flirtation, obscene, spam = response
+#       except: toxicity, severe_toxicity, identity_attack, insult, profanity, threat = response
+        would_have_been_deleted = []
+        would_have_been_suspicous = []
+        c.execute("SELECT * FROM moderation WHERE guild_id = ?", (str(ctx.guild.id),))
+        data = c.fetchone()
+        for i in response:
+            if i >= float(data[response.index(i)+4]):
+                would_have_been_deleted.append(tox.toxicity_names[response.index(i)])
+            elif i >= float(data[response.index(i)+4])-0.1:
+                would_have_been_suspicous.append(tox.toxicity_names[response.index(i)])
+        if would_have_been_deleted !=[]: embed = discord.Embed(title="Toxicity", description=f"Here are the different toxicity scores of the message\n***{message}***", color=discord.Color.red())
+        elif would_have_been_suspicous !=[] and would_have_been_deleted ==[]: embed = discord.Embed(title="Toxicity", description=f"Here are the different toxicity scores of the message\n***{message}***", color=discord.Color.orange())
+        else: embed = discord.Embed(title="Toxicity", description=f"Here are the different toxicity scores of the message\n***{message}***", color=discord.Color.green())
+        for i in response: embed.add_field(name=tox.toxicity_names[response.index(i)], value=f"{str( float(i)*100)}%", inline=False)
+        if would_have_been_deleted != []: embed.add_field(name="Would have been deleted", value=f"Yes, the message would have been deleted because of the following toxicity scores: **{'**, **'.join(would_have_been_deleted)}**", inline=False)
+        if would_have_been_suspicous != [] and would_have_been_deleted == []: embed.add_field(name="Would have been marked as suspicious", value=f"Yes, the message would have been marked as suspicious because of the following toxicity scores: {', '.join(would_have_been_suspicous)}", inline=False)
+        await ctx.respond(embed=embed)
+
+    @discord.slash_command(name="moderation_help", description="Get help with the moderation AI")
+    async def moderation_help(self, ctx: discord.ApplicationContext):
+        embed = discord.Embed(title="Moderation AI help", description="Here is a list of all the moderation commands", color=discord.Color.blurple())
+        for definition in tox.toxicity_definitions:
+            embed.add_field(name=tox.toxicity_names[tox.toxicity_definitions.index(definition)], value=definition, inline=False)
+        await ctx.respond(embed=embed, ephemeral=True)

code/config.py

@@ -15,7 +15,28 @@ connp = sqlite3.connect('../database/premium.db')
 cp = connp.cursor()
 
 c.execute('''CREATE TABLE IF NOT EXISTS data (guild_id text, channel_id text, api_key text, is_active boolean, max_tokens integer, temperature real, frequency_penalty real, presence_penalty real, uses_count_today integer, prompt_size integer, prompt_prefix text, tts boolean, pretend_to_be text, pretend_enabled boolean)''')
+#we delete the moderation table and create a new one, with all theese parameters as floats: TOXICITY: {result[0]}; SEVERE_TOXICITY: {result[1]}; IDENTITY ATTACK: {result[2]}; INSULT: {result[3]}; PROFANITY: {result[4]}; THREAT: {result[5]}; SEXUALLY EXPLICIT: {result[6]}; FLIRTATION: {result[7]}; OBSCENE: {result[8]}; SPAM: {result[9]}
+expected_columns = 14
+
 #we delete the moderation table and create a new one
-c.execute('''CREATE TABLE IF NOT EXISTS moderation (guild_id text, logs_channel_id text, is_enabled boolean, mod_role_id text)''')
+c.execute('''CREATE TABLE IF NOT EXISTS moderation (guild_id text, logs_channel_id text, is_enabled boolean, mod_role_id text, toxicity real, severe_toxicity real, identity_attack real, insult real, profanity real, threat real, sexually_explicit real, flirtation real, obscene real, spam real)''')
+c.execute("PRAGMA table_info(moderation)")
+result = c.fetchall()
+actual_columns = len(result)
+if actual_columns != expected_columns:
+    #we add the new columns
+    c.execute("ALTER TABLE moderation ADD COLUMN toxicity real")
+    c.execute("ALTER TABLE moderation ADD COLUMN severe_toxicity real")
+    c.execute("ALTER TABLE moderation ADD COLUMN identity_attack real")
+    c.execute("ALTER TABLE moderation ADD COLUMN insult real")
+    c.execute("ALTER TABLE moderation ADD COLUMN profanity real")
+    c.execute("ALTER TABLE moderation ADD COLUMN threat real")
+    c.execute("ALTER TABLE moderation ADD COLUMN sexually_explicit real")
+    c.execute("ALTER TABLE moderation ADD COLUMN flirtation real")
+    c.execute("ALTER TABLE moderation ADD COLUMN obscene real")
+    c.execute("ALTER TABLE moderation ADD COLUMN spam real")
+else:
+    print("Table already has the correct number of columns")
+    pass
 cp.execute('''CREATE TABLE IF NOT EXISTS data (user_id text, guild_id text, premium boolean)''')
 cp.execute('''CREATE TABLE IF NOT EXISTS channels (guild_id text, channel0 text, channel1 text, channel2 text, channel3 text, channel4 text)''')

code/toxicity.py

@@ -1,7 +1,21 @@
 from googleapiclient import discovery
 from config import perspective_api_key
-import json
 import re
+toxicity_names = ["toxicity", "severe_toxicity", "identity_attack", "insult", "profanity", "threat", "sexually_explicit", "flirtation", "obscene", "spam"]
+toxicity_definitions = [
+    "A rude, disrespectful, or unreasonable message that is likely to make people leave a discussion.",
+    "A very hateful, aggressive, disrespectful message or otherwise very likely to make a user leave a discussion or give up on sharing their perspective. This attribute is much less sensitive to more mild forms of toxicity, such as messages that include positive uses of curse words.",
+    "Negative or hateful messages targeting someone because of their identity.",
+    "Insulting, inflammatory, or negative messages towards a person or a group of people.",
+    "Swear words, curse words, or other obscene or profane language.",
+    "Describes an intention to inflict pain, injury, or violence against an individual or group.",
+    "Contains references to sexual acts, body parts, or other lewd content. \n **English only**",
+    "Pickup lines, complimenting appearance, subtle sexual innuendos, etc. \n **English only**",
+    "Obscene or vulgar language such as cursing. \n **English only**",
+    "Irrelevant and unsolicited commercial content. \n **English only**"
+]
+
+
 
 client = discovery.build("commentanalyzer",
                             "v1alpha1",
@@ -12,9 +26,18 @@ client = discovery.build("commentanalyzer",
 
 analyze_request = {
     'comment': {'text': ''},  # The text to analyze
-    'requestedAttributes': {'TOXICITY': {}},  # Requested attributes
+    #we will ask the following attributes to google: TOXICITY, SEVERE_TOXICITY, IDENTITY_ATTACK, INSULT, PRPFANITY, THREAT, SEXUALLY_EXPLICIT, FLIRTATION, OBSCENE, SPAM
-    #we will analyze the text in english, french & italian
+    'requestedAttributes': {'TOXICITY': {}, 'SEVERE_TOXICITY': {}, 'IDENTITY_ATTACK': {}, 'INSULT': {}, 'PROFANITY': {}, 'THREAT': {}, 'SEXUALLY_EXPLICIT': {}, 'FLIRTATION': {}, 'OBSCENE': {}, 'SPAM': {}},
-    'languages': ['en', 'fr', 'it'],
+    #we will analyze the text in any language automatically detected by google
+    'languages': [],
+    'doNotStore': 'true'  # We don't want google to store the data because of privacy reasons & the GDPR (General Data Protection Regulation, an EU law that protects the privacy of EU citizens and residents for data privacy and security purposes https://gdpr-info.eu/)
+}
+analyze_request_not_en = {
+    'comment': {'text': ''},  # The text to analyze
+    #we will ask the following attributes to google: TOXICITY, SEVERE_TOXICITY, IDENTITY_ATTACK, INSULT, PRPFANITY, THREAT, SEXUALLY_EXPLICIT, FLIRTATION, OBSCENE, SPAM
+    'requestedAttributes': {'TOXICITY': {}, 'SEVERE_TOXICITY': {}, 'IDENTITY_ATTACK': {}, 'INSULT': {}, 'PROFANITY': {}, 'THREAT': {}},
+    #we will analyze the text in any language automatically detected by google
+    'languages': [],
     'doNotStore': 'true'  # We don't want google to store the data because of privacy reasons & the GDPR (General Data Protection Regulation, an EU law that protects the privacy of EU citizens and residents for data privacy and security purposes https://gdpr-info.eu/)
 } 
 def get_toxicity(message: str):
@@ -28,22 +51,27 @@ def get_toxicity(message: str):
     message = re.sub(r'\~\~([^~]+)\~\~', r'\1', message)
     message = re.sub(r'\`([^`]+)\`', r'\1', message)
     message = re.sub(r'\`\`\`([^`]+)\`\`\`', r'\1', message) 
+    
+    #we try doing the request in english, but if we get 'errorType': 'LANGUAGE_NOT_SUPPORTED_BY_ATTRIBUTE' we try again with the analyze_request_not_en
+    try:
         analyze_request['comment']['text'] = message
         response = client.comments().analyze(body=analyze_request).execute()
-    return float(response['attributeScores']['TOXICITY']['summaryScore']['value'])
+    except:
+        analyze_request_not_en['comment']['text'] = message
+        response = client.comments().analyze(body=analyze_request_not_en).execute()
+    try: return [float(response['attributeScores']['TOXICITY']['summaryScore']['value']), float(response['attributeScores']['SEVERE_TOXICITY']['summaryScore']['value']), float(response['attributeScores']['IDENTITY_ATTACK']['summaryScore']['value']), float(response['attributeScores']['INSULT']['summaryScore']['value']), float(response['attributeScores']['PROFANITY']['summaryScore']['value']), float(response['attributeScores']['THREAT']['summaryScore']['value']), float(response['attributeScores']['SEXUALLY_EXPLICIT']['summaryScore']['value']), float(response['attributeScores']['FLIRTATION']['summaryScore']['value']), float(response['attributeScores']['OBSCENE']['summaryScore']['value']), float(response['attributeScores']['SPAM']['summaryScore']['value'])]
+    except: return [float(response['attributeScores']['TOXICITY']['summaryScore']['value']), float(response['attributeScores']['SEVERE_TOXICITY']['summaryScore']['value']), float(response['attributeScores']['IDENTITY_ATTACK']['summaryScore']['value']), float(response['attributeScores']['INSULT']['summaryScore']['value']), float(response['attributeScores']['PROFANITY']['summaryScore']['value']), float(response['attributeScores']['THREAT']['summaryScore']['value'])]
 
 #test part
 def test():
-    print(get_toxicity("Hello world"))
+    print("Testing toxicity.py...")
-    print(get_toxicity("You are a stupid bot I hate you!!!"))
+    print("Hello world:")
-    print(get_toxicity("Je suis un bot stupide, je vous déteste !!!"))
+    result = get_toxicity('Hello world')
-    print(get_toxicity("Ciao, come state?"))
+    try: print(f"TOXICITY: {result[0]}; SEVERE_TOXICITY: {result[1]}; IDENTITY ATTACK: {result[2]}; INSULT: {result[3]}; PROFANITY: {result[4]}; THREAT: {result[5]}; SEXUALLY EXPLICIT: {result[6]}; FLIRTATION: {result[7]}; OBSCENE: {result[8]}; SPAM: {result[9]}")
-    print(get_toxicity("Siete tutti degli scemi"))
+    except: print(f"TOXICITY: {result[0]}; SEVERE_TOXICITY: {result[1]}; IDENTITY ATTACK: {result[2]}; INSULT: {result[3]}; PROFANITY: {result[4]}; THREAT: {result[5]}")
-    print(get_toxicity("Siete tutti degli stupidi"))
+    print("HELLO WORLD GET ABSOLUTELY BUY MY NEW MERCH OMGGGGGGG:")
-    print(get_toxicity("Je n'aime pas les gens stupides"))
+    result = get_toxicity('HELLO WORLD GET ABSOLUTELY BUY MY NEW MERCH OMGGGGGGG')
-    #markdown removal test
+    try: print(f"TOXICITY: {result[0]}; SEVERE_TOXICITY: {result[1]}; IDENTITY ATTACK: {result[2]}; INSULT: {result[3]}; PROFANITY: {result[4]}; THREAT: {result[5]}; SEXUALLY EXPLICIT: {result[6]}; FLIRTATION: {result[7]}; OBSCENE: {result[8]}; SPAM: {result[9]}")
-    print(get_toxicity("You are all stupid"))
+    except: print(f"TOXICITY: {result[0]}; SEVERE_TOXICITY: {result[1]}; IDENTITY ATTACK: {result[2]}; INSULT: {result[3]}; PROFANITY: {result[4]}; THREAT: {result[5]}")
-    print(get_toxicity("You are all *s*t*u*p*i*d"))
-    print(print("*** you"))
 #uncomment the following line to test the code
 #test()