Compatibility issues with latest slack api
jamshid opened this issue · 1 comments
jamshid commented
Hi, thanks for providing this! With slack dropping some deprecated apis recently it was the only script I found that successfully downloaded DMs.
I made a few changes to handle some errors I encountered and added --debug and --all. Sorry I didn't make a real fork and pull request.
#!python3
# This script 'slack.py' downloads DMs. I can't believe there aren't more complete
# and official tools to do this.
# Slack recently change User Token authentiction and dropped some deprecated apis.
# This used old apis:
# https://gist.githubusercontent.com/Chandler/fb7a070f52883849de35/raw/f74bcec8c1c12ff2126212bb3d52e7f634e94b25/slack_history.py
# Tried using these changes but still had problems:
# https://gist.github.com/Chandler/fb7a070f52883849de35#gistcomment-3451274
# Finally found this which mostly worked just had to make some fixes and add --all:
# https://github.com/margaritageleta/slack-export-history
#
# Unfortunately it only grabs one-to-one DMs, not multi-user chats.
# It also doesn't grab channels. I think there are other tools that do that?
#
# python3 slack.py --token xoxp-4988244079-19823645074-1832659426481-blah --debug --all
#
# As the github README says, you have to Create New App and configure its User Token Scopes
# then Install App into your Workspace to get a User Token. The "OAuth and Permissions" link on left
# shows your "OAuth Tokens for Your Team" e.g. "xoxp-4988244079...".
# PS: I don't understand "for Your Team" because it's your user token that has access to
# your DMs so it should never be shared with your team? Anyway afaik the app is only
# visible by you.
#
# It creates a bunch of json files in a "slack-data" subdirectory. The files
# are named with the userid you you were chatting with.
#
# $ jq . 'chat_D5NPV744Q_(2900-2941).txt'
# ...
# {
# "user_id": "U220GNV2T",
# "user_name": "pray",
# "text": "Good. Rainy and cool. It's good to see Lena.",
# "ts": "1496765013.166083",
# "date": "2017-06-06 16:03:33"
# },
# ...
#
import os
import json
import shutil
import requests
import argparse
import time
import sys
import traceback
from datetime import datetime
try:
from pick import pick
except:
print('You need to "pip3 install pick" or always use --all and comment out the pick() call.')
sys.exit(1)
def auth(token):
try:
r = requests.post('https://slack.com/api/auth.test', data = {'token': token})
r.raise_for_status()
data = r.json()
if data['ok'] and data['ok']:
print(f"Successfully authenticated for team {data['team']} (ID {data['team_id']}) and user {data['user']} (ID {data['user_id']})")
return True
else:
print(f"Something went wrong. Error: {data['error']}")
return False
except Exception as e:
print(f'Something went wrong. Status code: {r.status_code}')
return False
def retrieve_data(endpoint, payload):
try:
r = requests.post(f'https://slack.com/api/{endpoint}', data = payload)
r.raise_for_status()
print(f'Data retrieved OK. Status code: {r.status_code}')
# print('!!!r.text=%s' % (r.text[:500]))
data = r.json()
# print('!!!%s' % (json.dumps(data, indent=4)[:500]))
if data['ok']:
with open(f'{endpoint}.json', 'w') as f:
json.dump(data, f, indent=4)
else:
print(f"Error: {data['error']}")
except Exception as e:
print(f'Something went wrong. Status code: {r.status_code}')
def fetch_users():
with open('users.list.json') as f:
users_dump = json.loads(f.read())
users = {}
for member in users_dump['members']:
# if not member['is_bot']:
users[member['id']] = {
'name': member['name'],
'real_name': member['profile']['real_name']
}
return users
def fetch_conversations():
with open('conversations.list.json') as f:
conversations_dump = json.loads(f.read())
conversations_dict = {}
conversations_list = []
for conver in conversations_dump['channels']:
if conver['is_im']:
#print('!!!conver=%s' % (conver))
conversations_dict[conver['id']] = {
'user_id': conver['user'],
'user_name': users[conver['user']]['name']
}
conversations_list.append(conver['id'])
return (conversations_dict, conversations_list)
""" These are only available if the types are specified on POST conversations.list?
if conver['is_mpim']:
channels[conver['id']] = {
'creator': conver['creator'],
}
if conver['is_channel']:
channels[conver['id']] = {
'creator': conver['creator'],
'is_private': conver['is_private']
}
"""
def fetch_message_data(payload):
r = data = None
back = 0
try:
# while there are older messages
while r == None or data['has_more']:
# and it is not the first request
if r != None:
# change the 'latest' argument to fetch older messages
payload['latest'] = data['messages'][-1]['ts']
r = requests.post(f'https://slack.com/api/conversations.history', data = payload)
r.raise_for_status()
print(f'Data retrieved OK. Status code: {r.status_code}')
time.sleep(1)
data = r.json()
if data['ok']:
messages = []
for message in data['messages']:
# print(u'!!!message=%s' % (message.keys()))
messages.append({
'user_id': message['user'] if 'user' in message else 'UNKNOWN',
'user_name': users[message['user']]['name'] if 'user' in message else message['username'] if 'username' in message else 'UNKNOWN',
'text': message['text'],
'ts': message['ts'],
'date': datetime.fromtimestamp(float(message['ts'])).strftime('%Y-%m-%d %H:%M:%S')
})
with open(f"chat_{payload['channel']}_({back}-{back + len(data['messages']) - 1}).txt", 'w') as f:
json.dump(messages, f, indent=4)
back += len(data['messages'])
else:
print(f"Error: {data['error']}")
except Exception as e:
print('Exception: %s' % (repr(e)))
traceback.print_exc()
print(f'Something went wrong. Status code: {r.status_code}')
sys.exit(1)
if __name__ == "__main__":
# Define parser to pass OAuth token
parser = argparse.ArgumentParser(description = 'Export Slack history')
parser.add_argument('--token', required = True, help = "OAuth Access Token")
parser.add_argument('--all', required = False, action='store_true', help = "whether to save DMs with all users")
parser.add_argument('--debug', required = False, action='store_true', help = "whether to show HTTP requests")
args = parser.parse_args()
if args.debug:
import logging
import contextlib
from http.client import HTTPConnection # py3
HTTPConnection.debuglevel = 5
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
# Do Auth Test to check user
if auth(args.token):
# Define the payload to do requests at Slack API
PAYLOAD = {
'token': args.token,
}
# Create a directory where to store the data
dir = 'slack-data'
if not os.path.exists(dir):
os.makedirs(dir)
os.chdir(dir)
# Retrieve users and conversations lists
retrieve_data('users.list', PAYLOAD)
users = fetch_users()
PAYLOAD['types'] = 'im'
retrieve_data('conversations.list', PAYLOAD)
# Select chat to export
title = 'Please the conversation to export: '
convers, options = fetch_conversations()
if args.all:
for id in options:
p = PAYLOAD.copy()
print(f'\nPreparing to export chat {id} ({convers[id]["user_name"]})...\n')
time.sleep(1)
p['channel'] = id
fetch_message_data(p)
time.sleep(1)
else:
option, index = pick([f"Chat {option} with {convers[option]['user_name']}" for option in options], title)
PAYLOAD['channel'] = options[index]
# Export chat
print('\nPreparing to export chat ...\n')
fetch_message_data(PAYLOAD)
else:
# Auth fail
pass
margaritageleta commented
Hey @jamshid thanks for the feedback! I encountered the same issue - didn't find any working slack scrapper after their API update so I decided to make it on my own. Actually, to dowload channel messages or multi-user chats you should make a workaround on the fetch_conversations() function (there is some code commented), but I didn't do that bc I just needed DM. Saying that, the code can be much improved yes. Actually, thanks for your extra things on the code, would be wonderful if you could make a pull req