
# Python version
from platform import python_version
print('Versão da Linguagem Python Usada Neste Jupyter Notebook:', python_version())

Versão da Linguagem Python Usada Neste Jupyter Notebook: 3.7.6


# https://pypi.org/project/boto3/
!pip install -q boto3


# https://pypi.org/project/spacy/
!pip install -q spacy


# Instala o dicionário
!python3 -m spacy download en_core_web_lg


# https://pypi.org/project/youtube_dl/
!pip install -q youtube-dl


# https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html
# https://pypi.org/project/awscli/
!pip install -q awscli


# Imports
import os
import json
import time
import boto3
import spacy
import numpy as np
import pandas as pd
import matplotlib as plt
import matplotlib.pyplot as plt
import seaborn as sns
from math import pi
from botocore.exceptions import ClientError
from datetime import date
import warnings
warnings.filterwarnings("ignore")
%matplotlib inline


# Dictionary with the URL of each video
dict_urls_youtube = {'luther_king': 'https://www.youtube.com/watch?v=I47Y6VHc3Ms',
                     'kennedy': 'https://www.youtube.com/watch?v=QAmHcdwKgtQ',
                     'churchill': 'https://www.youtube.com/watch?v=s_LncVnecLA',
                     'reagan': 'https://www.youtube.com/watch?v=5MDFX-dNtsM&t=6s',
                     'macarthur': 'https://www.youtube.com/watch?v=_42_aLGkRpg&t=19s'}


# Download the audio from the video
for url in list(dict_urls_youtube.values()):
    print(f'Fazendo Download do Vídeo: {url}')
    os.system(f'youtube-dl -x --audio-format "mp3" {url}')

Fazendo Download do Vídeo: https://www.youtube.com/watch?v=I47Y6VHc3Ms
Fazendo Download do Vídeo: https://www.youtube.com/watch?v=QAmHcdwKgtQ
Fazendo Download do Vídeo: https://www.youtube.com/watch?v=s_LncVnecLA
Fazendo Download do Vídeo: https://www.youtube.com/watch?v=5MDFX-dNtsM&t=6s
Fazendo Download do Vídeo: https://www.youtube.com/watch?v=_42_aLGkRpg&t=19s


# Create a DataFrame
df_audio_videos = pd.DataFrame.from_dict(dict_urls_youtube, orient = 'index', columns = ['youtube_urls'])


# View
df_audio_videos


# Rename the videos
for arquivo_audio in os.listdir('.'):
    
    if 'churchill' in arquivo_audio.lower():
        os.rename(arquivo_audio, 'churchill.mp3')
        df_audio_videos.at['churchill', 'filename'] = 'churchill.mp3'
    
    elif 'reagan' in arquivo_audio.lower():
        os.rename(arquivo_audio.lower(), 'reagan.mp3')
        df_audio_videos.at['reagan', 'filename'] = 'reagan.mp3'
    
    elif 'king' in arquivo_audio.lower():
        os.rename(arquivo_audio.lower(), 'king.mp3')
        df_audio_videos.at['luther_king', 'filename'] = 'king.mp3'
    
    elif 'macarthur' in arquivo_audio.lower():
        os.rename(arquivo_audio.lower(), 'macarthur.mp3')
        df_audio_videos.at['macarthur', 'filename'] = 'macarthur.mp3'
    
    elif 'kennedy' in arquivo_audio.lower():
        os.rename(arquivo_audio.lower(), 'kennedy.mp3')
        df_audio_videos.at['kennedy', 'filename'] = 'kennedy.mp3'


# View
df_audio_videos


# Set my credentials
aws_access_key_id = ''
aws_secret_access_key = ''
region_name = 'us-east-2'


# Create Session
session = boto3.session.Session(aws_access_key_id = aws_access_key_id,
                                aws_secret_access_key = aws_secret_access_key,
                                region_name = region_name)


# Create a connection with S3
s3 = boto3.client('s3')
client_s3 = boto3.client('s3')


# Identify the buckets
response = s3.list_buckets()
buckets = [bucket['Name'] for bucket in response['Buckets']]


# List the buckets
for bucket in buckets:
    print(bucket)

cf-templates-k86kpoh8b52x-us-east-2
design-aws-dsa
design-aws-dsa-log
design-aws-dsa-web
design-aws-dsa-webfiles
dsa-deploy-app
dsa-sistemas-mini-projeto5
sagemaker-studio-879456481532-3juvpt4pbqm
sagemaker-studio-879456481532-du2zpon7mfr
sagemaker-studio-8h3p453lu7q
sagemaker-us-east-2-879456481532
www.designawsdsa.com


# Define the bucket to upload files
bucket_name = 'dsa-sistemas-mini-projeto5'


# Upload the videos
for arquivo_audio in df_audio_videos.filename.values:
    print(arquivo_audio)
    client_s3.upload_file(arquivo_audio, bucket_name,  arquivo_audio)

king.mp3
kennedy.mp3
churchill.mp3
reagan.mp3
macarthur.mp3


# View
df_audio_videos


# Define the file URLs on S3
for index, row in df_audio_videos.iterrows():
    
    bucket_location = boto3.client('s3').get_bucket_location(Bucket = bucket_name)
    
    object_uri = f"s3://{bucket_name}/{row['filename']}"
    
    df_audio_videos.at[index, 'uri'] = object_uri
    
    print(object_uri)

s3://dsa-sistemas-mini-projeto5/king.mp3
s3://dsa-sistemas-mini-projeto5/kennedy.mp3
s3://dsa-sistemas-mini-projeto5/churchill.mp3
s3://dsa-sistemas-mini-projeto5/reagan.mp3
s3://dsa-sistemas-mini-projeto5/macarthur.mp3


# View
df_audio_videos


# Function to initialize the Amazon Transcribe job
def transcreve_audio(bucket, job_name, file_uri, wait_process = True):
    
    # Open the client
    client_transcribe = boto3.client('transcribe', region_name = 'us-east-2')
    
    # File URI
    print(file_uri)
    
    # Function to start transcription
    client_transcribe.start_transcription_job(TranscriptionJobName = job_name,
                                              Media = {'MediaFileUri': file_uri},
                                              MediaFormat = 'mp3',
                                              LanguageCode = 'en-US',
                                              OutputBucketName = bucket)
    
    # Conditional to wait the job execution
    if wait_process:
        while True:
            status = client_transcribe.get_transcription_job(TranscriptionJobName = job_name)
            if status['TranscriptionJob']['TranscriptionJobStatus'] in ['COMPLETED', 'FAILED']:
                break
            print("Processamento em Andamento.")
            time.sleep(20)

        print('Transcrição Concluída!')
        return status


# Define the key to use on the transcription job
key = "dsa-mini-projeto5"


# Run the transcription function
for index, row in df_audio_videos.iterrows():    
    
    # Print
    print(f'{index}_speech_{key}')
    print(row.uri)
    
    # Transcribe text from the audio files
    transcreve_audio(bucket_name, f'{index}_speech_{key}', row.uri, wait_process = False)
    
    # Save results to the DataFrame
    df_audio_videos.at[index, 'transcription_url'] = f"https://{bucket_name}.s3.amazonaws.com/{index}_speech_{key}.json"
    df_audio_videos.at[index, 'json_transcription'] = f"{index}_speech_{key}.json"

luther_king_speech_dsa-mini-projeto5
s3://dsa-sistemas-mini-projeto5/king.mp3
s3://dsa-sistemas-mini-projeto5/king.mp3
kennedy_speech_dsa-mini-projeto5
s3://dsa-sistemas-mini-projeto5/kennedy.mp3
s3://dsa-sistemas-mini-projeto5/kennedy.mp3
churchill_speech_dsa-mini-projeto5
s3://dsa-sistemas-mini-projeto5/churchill.mp3
s3://dsa-sistemas-mini-projeto5/churchill.mp3
reagan_speech_dsa-mini-projeto5
s3://dsa-sistemas-mini-projeto5/reagan.mp3
s3://dsa-sistemas-mini-projeto5/reagan.mp3
macarthur_speech_dsa-mini-projeto5
s3://dsa-sistemas-mini-projeto5/macarthur.mp3
s3://dsa-sistemas-mini-projeto5/macarthur.mp3


# View
df_audio_videos


# Check the Amazon Transcribe jobs with COMPLETED status
client_transcribe = boto3.client('transcribe', region_name = 'us-east-2')
resposta = client_transcribe.list_transcription_jobs(Status = 'COMPLETED')
resposta

{'Status': 'COMPLETED',
 'TranscriptionJobSummaries': [{'TranscriptionJobName': 'macarthur_speech_dsa-mini-projeto5',
   'CreationTime': datetime.datetime(2021, 4, 17, 17, 12, 5, 470000, tzinfo=tzlocal()),
   'StartTime': datetime.datetime(2021, 4, 17, 17, 12, 5, 494000, tzinfo=tzlocal()),
   'CompletionTime': datetime.datetime(2021, 4, 17, 17, 17, 6, 318000, tzinfo=tzlocal()),
   'LanguageCode': 'en-US',
   'TranscriptionJobStatus': 'COMPLETED',
   'OutputLocationType': 'CUSTOMER_BUCKET'},
  {'TranscriptionJobName': 'reagan_speech_dsa-mini-projeto5',
   'CreationTime': datetime.datetime(2021, 4, 17, 17, 12, 4, 880000, tzinfo=tzlocal()),
   'StartTime': datetime.datetime(2021, 4, 17, 17, 12, 4, 899000, tzinfo=tzlocal()),
   'CompletionTime': datetime.datetime(2021, 4, 17, 17, 15, 39, 965000, tzinfo=tzlocal()),
   'LanguageCode': 'en-US',
   'TranscriptionJobStatus': 'COMPLETED',
   'OutputLocationType': 'CUSTOMER_BUCKET'},
  {'TranscriptionJobName': 'churchill_speech_dsa-mini-projeto5',
   'CreationTime': datetime.datetime(2021, 4, 17, 17, 12, 4, 411000, tzinfo=tzlocal()),
   'StartTime': datetime.datetime(2021, 4, 17, 17, 12, 4, 441000, tzinfo=tzlocal()),
   'CompletionTime': datetime.datetime(2021, 4, 17, 17, 14, 16, 490000, tzinfo=tzlocal()),
   'LanguageCode': 'en-US',
   'TranscriptionJobStatus': 'COMPLETED',
   'OutputLocationType': 'CUSTOMER_BUCKET'},
  {'TranscriptionJobName': 'kennedy_speech_dsa-mini-projeto5',
   'CreationTime': datetime.datetime(2021, 4, 17, 17, 12, 3, 896000, tzinfo=tzlocal()),
   'StartTime': datetime.datetime(2021, 4, 17, 17, 12, 3, 916000, tzinfo=tzlocal()),
   'CompletionTime': datetime.datetime(2021, 4, 17, 17, 14, 27, 192000, tzinfo=tzlocal()),
   'LanguageCode': 'en-US',
   'TranscriptionJobStatus': 'COMPLETED',
   'OutputLocationType': 'CUSTOMER_BUCKET'},
  {'TranscriptionJobName': 'luther_king_speech_dsa-mini-projeto5',
   'CreationTime': datetime.datetime(2021, 4, 17, 17, 12, 3, 383000, tzinfo=tzlocal()),
   'StartTime': datetime.datetime(2021, 4, 17, 17, 12, 3, 407000, tzinfo=tzlocal()),
   'CompletionTime': datetime.datetime(2021, 4, 17, 17, 14, 59, 211000, tzinfo=tzlocal()),
   'LanguageCode': 'en-US',
   'TranscriptionJobStatus': 'COMPLETED',
   'OutputLocationType': 'CUSTOMER_BUCKET'}],
 'ResponseMetadata': {'RequestId': '64ca3e26-7b56-47bf-a0fd-a091a50cd456',
  'HTTPStatusCode': 200,
  'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',
   'date': 'Sun, 18 Apr 2021 00:17:33 GMT',
   'x-amzn-requestid': '64ca3e26-7b56-47bf-a0fd-a091a50cd456',
   'content-length': '1326',
   'connection': 'keep-alive'},
  'RetryAttempts': 0}}


# Function to get the text from the JSOn file generated by Amazon Transcribe
def get_texto(bucket, key):
    
    # Connect to S3
    s3 = boto3.client('s3', region_name = 'us-east-2')
    
    # Generate the object with the bucket connection
    object = s3.get_object(Bucket = bucket, Key = key)
    
    # Serialize the object
    serializedObject = object['Body'].read()
    
    # Load the JSON data
    data = json.loads(serializedObject)
    
    # Return the transcription
    return data.get('results').get('transcripts')[0].get('transcript')


# Read the original transcription from the JSON file
with open('dados/transcripts_originais.json', 'r') as f:
    original_transcriptions = json.load(f)


# View
original_transcriptions

{'churchill': 'When, a week ago today, I asked the House to fix this afternoon as the occasion for a statement, I feared it would be my hard lot to announce the greatest military disaster in our long history. I thought-and some good judges agreed with me-that perhaps 20,000 or 30,000 men might be re-embarked. But it certainly seemed that the whole of the French First Army and the whole of the British Expeditionary Force north of the Amiens-Abbeville gap would be broken up in the open field or else would have to capitulate for lack of food and ammunition. These were the hard and heavy tidings for which I called upon the House and the nation to prepare themselves a week ago. The whole root and core and brain of the British Army, on which and around which we were to build, and are to build, the great British Armies in the later years of the war, seemed about to perish upon the field or to be led into an ignominious and starving captivity.The enemy attacked on all sides with great strength and fierceness, and their main power, the power of their far more numerous Air Force, was thrown into the battle or else concentrated upon Dunkirk and the beaches. Pressing in upon the narrow exit, both from the east and from the west, the enemy began to fire with cannon upon the beaches by which alone the shipping could approach or depart. They sowed magnetic mines in the channels and seas; they sent repeated waves of hostile aircraft, sometimes more than a hundred strong in one formation, to cast their bombs upon the single pier that remained, and upon the sand dunes upon which the troops had their eyes for shelter. Their U-boats, one of which was sunk, and their motor launches took their toll of the vast traffic which now began. For four or five days an intense struggle reigned. All their armored divisions-or what Was left of them-together with great masses of infantry and artillery, hurled themselves in vain upon the ever-narrowing, ever-contracting appendix within which the British and French Armies fought. Meanwhile, the Royal Navy, with the willing help of countless merchant seamen, strained every nerve to embark the British and Allied troops; 220 light warships and 650 other vessels were engaged. They had to operate upon the difficult coast, often in adverse weather, under an almost ceaseless hail of bombs and an increasing concentration of artillery fire. Nor were the seas, as I have said, themselves free from mines and torpedoes. It was in conditions such as these that our men carried on, with little or no rest, for days and nights on end, making trip after trip across the dangerous waters, bringing with them always men whom they had rescued. The numbers they have brought back are the measure of their devotion and their courage. The hospital ships, which brought off many thousands of British and French wounded, being so plainly marked were a special target for Nazi bombs; but the men and women on board them never faltered in their duty. Meanwhile, the Royal Air Force, which had already been intervening in the battle, so far as its range would allow, from home bases, now used part of its main metropolitan fighter strength, and struck at the German bombers and at the fighters which in large numbers protected them. This struggle was protracted and fierce. Suddenly the scene has cleared, the crash and thunder has for the moment-but only for the moment-died away. A miracle of deliverance, achieved by valor, by perseverance, by perfect discipline, by faultless service, by resource, by skill, by unconquerable fidelity, is manifest to us all. The enemy was hurled back by the retreating British and French troops. He was so roughly handled that he did not hurry their departure seriously. We must be very careful not to assign to this deliverance the attributes of a victory. Wars are not won by evacuations. But there was a victory inside this deliverance, which should be noted. It was gained by the Air Force. Many of our soldiers coming back have not seen the Air Force at work; they saw only the bombers which escaped its protective attack. They underrate its achievements. I have heard much talk of this; that is why I go out of my way to say this. I will tell you about it. This was a great trial of strength between the British and German Air Forces. Can you conceive a greater objective for the Germans in the air than to make evacuation from these beaches impossible, and to sink all these ships which were displayed, almost to the extent of thousands? Could there have been an objective of greater military importance and significance for the whole purpose of the war than this? They tried hard, and they were beaten back; they were frustrated in their task. We got the Army away; and they have paid fourfold for any losses which they have inflicted. When we consider how much greater would be our advantage in defending the air above this Island against an overseas attack, I must say that I find in these facts a sure basis upon which practical and reassuring thoughts may rest. I will pay my tribute to these young airmen. The great French Army was very largely, for the time being, cast back and disturbed by the onrush of a few thousands of armored vehicles. May it not also be that the cause of civilization itself will be defended by the skill and devotion of a few thousand airmen? There never has been, I suppose, in all the world, in all the history of war, such an opportunity for youth. The Knights of the Round Table, the Crusaders, all fall back into the past-not only distant but prosaic; these young men, going forth every morn to guard their native land and all that we stand for, holding in their hands these instruments of colossal and shattering power, of whom it may be said that every morn brought forth a noble chance and every chance brought forth a noble knight, deserve our gratitude, as do all the brave men who, in so many ways and on so many occasions, are ready, and continue ready to give life and all for their native land. Nevertheless, our thankfulness at the escape of our Army and so many men, whose loved ones have passed through an agonizing week, must not blind us to the fact that what has happened in France and Belgium is a colossal military disaster. The French Army has been weakened, the Belgian Army has been lost, a large part of those fortified lines upon which so much faith had been reposed is gone, many valuable mining districts and factories have passed into the enemy’s possession, the whole of the Channel ports are in his hands, with all the tragic consequences that follow from that, and we must expect another blow to be struck almost immediately at us or at France. We are told that Herr Hitler has a plan for invading the British Isles. This has often been thought of before. When Napoleon lay at Boulogne for a year with his flat-bottomed boats and his Grand Army, he was told by someone. “There are bitter weeds in England.” There are certainly a great many more of them since the British Expeditionary Force returned. I have, myself, full confidence that if all do their duty, if nothing is neglected, and if the best arrangements are made, as they are being made, we shall prove ourselves once again able to defend our Island home, to ride out the storm of war, and to outlive the menace of tyranny, if necessary for years, if necessary alone. At any rate, that is what we are going to try to do. That is the resolve of His Majesty’s Government-every man of them. That is the will of Parliament and the nation. The British Empire and the French Republic, linked together in their cause and in their need, will defend to the death their native soil, aiding each other like good comrades to the utmost of their strength. We shall go on to the end, we shall fight in France, we shall fight on the seas and oceans, we shall fight with growing confidence and growing strength in the air, we shall defend our Island, whatever the cost may be, we shall fight on the beaches, we shall fight on the landing grounds, we shall fight in the fields and in the streets, we shall fight in the hills; we shall never surrender, and even if, which I do not for a moment believe, this Island or a large part of it were subjugated and starving, then our Empire beyond the seas, armed and guarded by the British Fleet, would carry on the struggle, until, in God’s good time, the New World, with all its power and might, steps forth to the rescue and the liberation of the old.',
 'reagan': 'Thank you. Thank you, very much. Chancellor Kohl, Governing Mayor Diepgen, ladies and gentlemen: Twenty four years ago, President John F. Kennedy visited Berlin, and speaking to the people of this city and the world at the city hall. Well since then two other presidents have come, each in his turn to Berlin. And today, I, myself, make my second visit to your city. We come to Berlin, we American Presidents, because it’s our duty to speak in this place of freedom. But I must confess, we’re drawn here by other things as well; by the feeling of history in this city — more than 500 years older than our own nation; by the beauty of the Grunewald and the Tiergarten; most of all, by your courage and determination. Perhaps the composer, Paul Linke, understood something about American Presidents. You see, like so many Presidents before me, I come here today because wherever I go, whatever I do: “Ich hab noch einen Koffer in Berlin” [I still have a suitcase in Berlin.] Our gathering today is being broadcast throughout Western Europe and North America. I understand that it is being seen and heard as well in the East. To those listening throughout Eastern Europe, I extend my warmest greetings and the good will of the American people. To those listening in East Berlin, a special word: Although I cannot be with you, I address my remarks to you just as surely as to those standing here before me. For I join you, as I join your fellow countrymen in the West, in this firm, this unalterable belief: Es gibt nur ein Berlin. [There is only one Berlin.] Behind me stands a wall that encircles the free sectors of this city, part of a vast system of barriers that divides the entire continent of Europe. From the Baltic South, those barriers cut across Germany in a gash of barbed wire, concrete, dog runs, and guard towers. Farther south, there may be no visible, no obvious wall. But there remain armed guards and checkpoints all the same — still a restriction on the right to travel, still an instrument to impose upon ordinary men and women the will of a totalitarian state. Yet, it is here in Berlin where the wall emerges most clearly; here, cutting across your city, where the news photo and the television screen have imprinted this brutal division of a continent upon the mind of the world. Standing before the Brandenburg Gate, every man is a German separated from his fellow men. Every man is a Berliner, forced to look upon a scar. President Von WeizsÃ¤cker has said, “The German question is open as long as the Brandenburg Gate is closed.” Well today — today I say: As long as this gate is closed, as long as this scar of a wall is permitted to stand, it is not the German question alone that remains open, but the question of freedom for all mankind. Yet, I do not come here to lament. For I find in Berlin a message of hope, even in the shadow of this wall, a message of triumph. In this season of spring in 1945, the people of Berlin emerged from their air-raid shelters to find devastation. Thousands of miles away, the people of the United States reached out to help. And in 1947 Secretary of State — as you’ve been told — George Marshall announced the creation of what would become known as the Marshall Plan. Speaking precisely 40 years ago this month, he said: “Our policy is directed not against any country or doctrine, but against hunger, poverty, desperation, and chaos.” In the Reichstag a few moments ago, I saw a display commemorating this 40th anniversary of the Marshall Plan. I was struck by a sign — the sign on a burnt-out, gutted structure that was being rebuilt. I understand that Berliners of my own generation can remember seeing signs like it dotted throughout the western sectors of the city. The sign read simply: “The Marshall Plan is helping here to strengthen the free world.” A strong, free world in the West — that dream became real. Japan rose from ruin to become an economic giant. Italy, France, Belgium — virtually every nation in Western Europe saw political and economic rebirth; the European Community was founded. In West Germany and here in Berlin, there took place an economic miracle, the Wirtschaftswunder. Adenauer, Erhard, Reuter, and other leaders understood the practical importance of liberty — that just as truth can flourish only when the journalist is given freedom of speech, so prosperity can come about only when the farmer and businessman enjoy economic freedom. The German leaders — the German leaders reduced tariffs, expanded free trade, lowered taxes. From 1950 to 1960 alone, the standard of living in West Germany and Berlin doubled. Where four decades ago there was rubble, today in West Berlin there is the greatest industrial output of any city in Germany: busy office blocks, fine homes and apartments, proud avenues, and the spreading lawns of parkland. Where a city’s culture seemed to have been destroyed, today there are two great universities, orchestras and an opera, countless theaters, and museums. Where there was want, today there’s abundance — food, clothing, automobiles — the wonderful goods of the Kudamm.¹ From devastation, from utter ruin, you Berliners have, in freedom, rebuilt a city that once again ranks as one of the greatest on earth. Now the Soviets may have had other plans. But my friends, there were a few things the Soviets didn’t count on: Berliner Herz, Berliner Humor, ja, und Berliner Schnauze. [Berliner heart, Berliner humor, yes, and a Berliner Schnauze.²] In the 1950s — In the 1950s Khrushchev predicted: “We will bury you.” But in the West today, we see a free world that has achieved a level of prosperity and well-being unprecedented in all human history. In the Communist world, we see failure, technological backwardness, declining standards of health, even want of the most basic kind — too little food. Even today, the Soviet Union still cannot feed itself. After these four decades, then, there stands before the entire world one great and inescapable conclusion: Freedom leads to prosperity. Freedom replaces the ancient hatreds among the nations with comity and peace. Freedom is the victor. And now — now the Soviets themselves may, in a limited way, be coming to understand the importance of freedom. We hear much from Moscow about a new policy of reform and openness. Some political prisoners have been released. Certain foreign news broadcasts are no longer being jammed. Some economic enterprises have been permitted to operate with greater freedom from state control. Are these the beginnings of profound changes in the Soviet state? Or are they token gestures intended to raise false hopes in the West, or to strengthen the Soviet system without changing it? We welcome change and openness; for we believe that freedom and security go together, that the advance of human liberty — the advance of human liberty can only strengthen the cause of world peace. There is one sign the Soviets can make that would be unmistakable, that would advance dramatically the cause of freedom and peace. General Secretary Gorbachev, if you seek peace, if you seek prosperity for the Soviet Union and Eastern Europe, if you seek liberalization: Come here to this gate. Mr. Gorbachev, open this gate. Mr. Gorbachev — Mr. Gorbachev, tear down this wall! I understand the fear of war and the pain of division that afflict this continent, and I pledge to you my country’s efforts to help overcome these burdens. To be sure, we in the West must resist Soviet expansion. So, we must maintain defenses of unassailable strength. Yet we seek peace; so we must strive to reduce arms on both sides. Beginning 10 years ago, the Soviets challenged the Western alliance with a grave new threat, hundreds of new and more deadly SS-20 nuclear missiles capable of striking every capital in Europe. The Western alliance responded by committing itself to a counter-deployment (unless the Soviets agreed to negotiate a better solution) — namely, the elimination of such weapons on both sides. For many months, the Soviets refused to bargain in earnestness. As the alliance, in turn, prepared to go forward with its counter-deployment, there were difficult days, days of protests like those during my 1982 visit to this city; and the Soviets later walked away from the table. But through it all, the alliance held firm. And I invite those who protested then — I invite those who protest today — to mark this fact: Because we remained strong, the Soviets came back to the table. Because we remained strong, today we have within reach the possibility, not merely of limiting the growth of arms, but of eliminating, for the first time, an entire class of nuclear weapons from the face of the earth. As I speak, NATO ministers are meeting in Iceland to review the progress of our proposals for eliminating these weapons. At the talks in Geneva, we have also proposed deep cuts in strategic offensive weapons. And the Western allies have likewise made far-reaching proposals to reduce the danger of conventional war and to place a total ban on chemical weapons. While we pursue these arms reductions, I pledge to you that we will maintain the capacity to deter Soviet aggression at any level at which it might occur. And in cooperation with many of our allies, the United States is pursuing the Strategic Defense Initiative — research to base deterrence not on the threat of offensive retaliation, but on defenses that truly defend; on systems, in short, that will not target populations, but shield them. By these means we seek to increase the safety of Europe and all the world. But we must remember a crucial fact: East and West do not mistrust each other because we are armed; we are armed because we mistrust each other. And our differences are not about weapons but about liberty. When President Kennedy spoke at the City Hall those 24 years ago, freedom was encircled; Berlin was under siege. And today, despite all the pressures upon this city, Berlin stands secure in its liberty. And freedom itself is transforming the globe. In the Philippines, in South and Central America, democracy has been given a rebirth. Throughout the Pacific, free markets are working miracle after miracle of economic growth. In the industrialized nations, a technological revolution is taking place, a revolution marked by rapid, dramatic advances in computers and telecommunications. In Europe, only one nation and those it controls refuse to join the community of freedom. Yet in this age of redoubled economic growth, of information and innovation, the Soviet Union faces a choice: It must make fundamental changes, or it will become obsolete. Today, thus, represents a moment of hope. We in the West stand ready to cooperate with the East to promote true openness, to break down barriers that separate people, to create a safer, freer world. And surely there is no better place than Berlin, the meeting place of East and West, to make a start. Free people of Berlin: Today, as in the past, the United States stands for the strict observance and full implementation of all parts of the Four Power Agreement of 1971. Let us use this occasion, the 750th anniversary of this city, to usher in a new era, to seek a still fuller, richer life for the Berlin of the future. Together, let us maintain and develop the ties between the Federal Republic and the Western sectors of Berlin, which is permitted by the 1971 agreement. And I invite Mr. Gorbachev: Let us work to bring the Eastern and Western parts of the city closer together, so that all the inhabitants of all Berlin can enjoy the benefits that come with life in one of the great cities of the world. To open Berlin still further to all Europe, East and West, let us expand the vital air access to this city, finding ways of making commercial air service to Berlin more convenient, more comfortable, and more economical. We look to the day when West Berlin can become one of the chief aviation hubs in all central Europe. With — With our French — With our French and British partners, the United States is prepared to help bring international meetings to Berlin. It would be only fitting for Berlin to serve as the site of United Nations meetings, or world conferences on human rights and arms control, or other issues that call for international cooperation. There is no better way to establish hope for the future than to enlighten young minds, and we would be honored to sponsor summer youth exchanges, cultural events, and other programs for young Berliners from the East. Our French and British friends, I’m certain, will do the same. And it’s my hope that an authority can be found in East Berlin to sponsor visits from young people of the Western sectors. One final proposal, one close to my heart: Sport represents a source of enjoyment and ennoblement, and you may have noted that the Republic of Korea — South Korea — has offered to permit certain events of the 1988 Olympics to take place in the North. International sports competitions of all kinds could take place in both parts of this city. And what better way to demonstrate to the world the openness of this city than to offer in some future year to hold the Olympic games here in Berlin, East and West. In these four decades, as I have said, you Berliners have built a great city. You’ve done so in spite of threats — the Soviet attempts to impose the East-mark, the blockade. Today the city thrives in spite of the challenges implicit in the very presence of this wall. What keeps you here? Certainly there’s a great deal to be said for your fortitude, for your defiant courage. But I believe there’s something deeper, something that involves Berlin’s whole look and feel and way of life — not mere sentiment. No one could live long in Berlin without being completely disabused of illusions. Something, instead, that has seen the difficulties of life in Berlin but chose to accept them, that continues to build this good and proud city in contrast to a surrounding totalitarian presence, that refuses to release human energies or aspirations, something that speaks with a powerful voice of affirmation, that says “yes” to this city, yes to the future, yes to freedom. In a word, I would submit that what keeps you in Berlin — is “love.” Love both profound and abiding. Perhaps this gets to the root of the matter, to the most fundamental distinction of all between East and West. The totalitarian world produces backwardness because it does such violence to the spirit, thwarting the human impulse to create, to enjoy, to worship. The totalitarian world finds even symbols of love and of worship an affront. Years ago, before the East Germans began rebuilding their churches, they erected a secular structure: the television tower at Alexander Platz. Virtually ever since, the authorities have been working to correct what they view as the tower’s one major flaw: treating the glass sphere at the top with paints and chemicals of every kind. Yet even today when the sun strikes that sphere, that sphere that towers over all Berlin, the light makes the sign of the cross. There in Berlin, like the city itself, symbols of love, symbols of worship, cannot be suppressed. As I looked out a moment ago from the Reichstag, that embodiment of German unity, I noticed words crudely spray-painted upon the wall, perhaps by a young Berliner (quote): “This wall will fall. Beliefs become reality.” Yes, across Europe, this wall will fall, for it cannot withstand faith; it cannot withstand truth. The wall cannot withstand freedom. And I would like, before I close, to say one word. I have read, and I have been questioned since I’ve been here about certain demonstrations against my coming. And I would like to say just one thing, and to those who demonstrate so. I wonder if they have ever asked themselves that if they should have the kind of government they apparently seek, no one would ever be able to do what they’re doing again. Thank you and God bless you all. Thank you.',
 'luther_king': 'I am happy to join with you today in what will go down in history as the greatest demonstration for freedom in the history of our nation. Five score years ago, a great American, in whose symbolic shadow we stand today, signed the Emancipation Proclamation. This momentous decree came as a great beacon light of hope to millions of Negro slaves who had been seared in the flames of withering injustice. It came as a joyous daybreak to end the long night of their captivity. But one hundred years later, the Negro still is not free. One hundred years later, the life of the Negro is still sadly crippled by the manacles of segregation and the chains of discrimination. One hundred years later, the Negro lives on a lonely island of poverty in the midst of a vast ocean of material prosperity. One hundred years later, the Negro is still languished in the corners of American society and finds himself an exile in his own land. And so we’ve come here today to dramatize a shameful condition. In a sense we’ve come to our nation’s capital to cash a check. When the architects of our republic wrote the magnificent words of the Constitution and the Declaration of Independence, they were signing a promissory note to which every American was to fall heir. This note was a promise that all men, yes, black men as well as white men, would be guaranteed the “unalienable Rights” of “Life, Liberty and the pursuit of Happiness.” It is obvious today that America has defaulted on this promissory note, insofar as her citizens of color are concerned. Instead of honoring this sacred obligation, America has given the Negro people a bad check, a check which has come back marked “insufficient funds.” But we refuse to believe that the bank of justice is bankrupt. We refuse to believe that there are insufficient funds in the great vaults of opportunity of this nation. And so, we’ve come to cash this check, a check that will give us upon demand the riches of freedom and the security of justice. We have also come to this hallowed spot to remind America of the fierce urgency of Now. This is no time to engage in the luxury of cooling off or to take the tranquilizing drug of gradualism. Now is the time to make real the promises of democracy. Now is the time to rise from the dark and desolate valley of segregation to the sunlit path of racial justice. Now is the time to lift our nation from the quicksands of racial injustice to the solid rock of brotherhood. Now is the time to make justice a reality for all of God’s children. It would be fatal for the nation to overlook the urgency of the moment. This sweltering summer of the Negro’s legitimate discontent will not pass until there is an invigorating autumn of freedom and equality. Nineteen sixty-three is not an end, but a beginning. And those who hope that the Negro needed to blow off steam and will now be content will have a rude awakening if the nation returns to business as usual. And there will be neither rest nor tranquility in America until the Negro is granted his citizenship rights. The whirlwinds of revolt will continue to shake the foundations of our nation until the bright day of justice emerges. But there is something that I must say to my people, who stand on the warm threshold which leads into the palace of justice: In the process of gaining our rightful place, we must not be guilty of wrongful deeds. Let us not seek to satisfy our thirst for freedom by drinking from the cup of bitterness and hatred. We must forever conduct our struggle on the high plane of dignity and discipline. We must not allow our creative protest to degenerate into physical violence. Again and again, we must rise to the majestic heights of meeting physical force with soul force. The marvelous new militancy which has engulfed the Negro community must not lead us to a distrust of all white people, for many of our white brothers, as evidenced by their presence here today, have come to realize that their destiny is tied up with our destiny. And they have come to realize that their freedom is inextricably bound to our freedom. We cannot walk alone. And as we walk, we must make the pledge that we shall always march ahead. We cannot turn back. There are those who are asking the devotees of civil rights, “When will you be satisfied?” We can never be satisfied as long as the Negro is the victim of the unspeakable horrors of police brutality. We can never be satisfied as long as our bodies, heavy with the fatigue of travel, cannot gain lodging in the motels of the highways and the hotels of the cities. We cannot be satisfied as long as the negro’s basic mobility is from a smaller ghetto to a larger one. We can never be satisfied as long as our children are stripped of their self-hood and robbed of their dignity by a sign stating: “For Whites Only.” We cannot be satisfied as long as a Negro in Mississippi cannot vote and a Negro in New York believes he has nothing for which to vote. No, no, we are not satisfied, and we will not be satisfied until “justice rolls down like waters, and righteousness like a mighty stream.” I am not unmindful that some of you have come here out of great trials and tribulations. Some of you have come fresh from narrow jail cells. And some of you have come from areas where your quest — quest for freedom left you battered by the storms of persecution and staggered by the winds of police brutality. You have been the veterans of creative suffering. Continue to work with the faith that unearned suffering is redemptive. Go back to Mississippi, go back to Alabama, go back to South Carolina, go back to Georgia, go back to Louisiana, go back to the slums and ghettos of our northern cities, knowing that somehow this situation can and will be changed. Let us not wallow in the valley of despair, I say to you today, my friends. And so even though we face the difficulties of today and tomorrow, I still have a dream. It is a dream deeply rooted in the American dream. I have a dream that one day this nation will rise up and live out the true meaning of its creed: “We hold these truths to be self-evident, that all men are created equal.” I have a dream that one day on the red hills of Georgia, the sons of former slaves and the sons of former slave owners will be able to sit down together at the table of brotherhood. I have a dream that one day even the state of Mississippi, a state sweltering with the heat of injustice, sweltering with the heat of oppression, will be transformed into an oasis of freedom and justice. I have a dream that my four little children will one day live in a nation where they will not be judged by the color of their skin but by the content of their character. I have a dream today! I have a dream that one day, down in Alabama, with its vicious racists, with its governor having his lips dripping with the words of “interposition” and “nullification” — one day right there in Alabama little black boys and black girls will be able to join hands with little white boys and white girls as sisters and brothers. I have a dream today! I have a dream that one day every valley shall be exalted, and every hill and mountain shall be made low, the rough places will be made plain, and the crooked places will be made straight; “and the glory of the Lord shall be revealed and all flesh shall see it together.”² This is our hope, and this is the faith that I go back to the South with. With this faith, we will be able to hew out of the mountain of despair a stone of hope. With this faith, we will be able to transform the jangling discords of our nation into a beautiful symphony of brotherhood. With this faith, we will be able to work together, to pray together, to struggle together, to go to jail together, to stand up for freedom together, knowing that we will be free one day. And this will be the day — this will be the day when all of God’s children will be able to sing with new meaning: My country ’tis of thee, sweet land of liberty, of thee I sing. Land where my fathers died, land of the Pilgrim’s pride, From every mountainside, let freedom ring! And if America is to be a great nation, this must become true.  And so let freedom ring from the prodigious hilltops of New Hampshire. Let freedom ring from the mighty mountains of New York. Let freedom ring from the heightening Alleghenies of Pennsylvania. Let freedom ring from the snow-capped Rockies of Colorado. Let freedom ring from the curvaceous slopes of California. But not only that: Let freedom ring from Stone Mountain of Georgia. Let freedom ring from Lookout Mountain of Tennessee. Let freedom ring from every hill and molehill of Mississippi. From every mountainside, let freedom ring. And when this happens, when we allow freedom ring, when we let it ring from every village and every hamlet, from every state and every city, we will be able to speed up that day when all of God’s children, black men and white men, Jews and Gentiles, Protestants and Catholics, will be able to join hands and sing in the words of the old Negro spiritual: Free at last! Free at last! Thank God Almighty, we are free at last!',
 'macarthur': 'General Westmoreland, General Grove, distinguished guests, and gentlemen of the Corps! As I was leaving the hotel this morning, a doorman asked me, “Where are you bound for, General?” And when I replied, “West Point,” he remarked, “Beautiful place. Have you ever been there before?” No human being could fail to be deeply moved by such a tribute as this [Thayer Award]. Coming from a profession I have served so long, and a people I have loved so well, it fills me with an emotion I cannot express. But this award is not intended primarily to honor a personality, but to symbolize a great moral code — the code of conduct and chivalry of those who guard this beloved land of culture and ancient descent. That is the animation of this medallion. For all eyes and for all time, it is an expression of the ethics of the American soldier. That I should be integrated in this way with so noble an ideal arouses a sense of pride and yet of humility which will be with me always Duty, Honor, Country: Those three hallowed words reverently dictate what you ought to be, what you can be, what you will be. They are your rallying points: to build courage when courage seems to fail; to regain faith when there seems to be little cause for faith; to create hope when hope becomes forlorn. Unhappily, I possess neither that eloquence of diction, that poetry of imagination, nor that brilliance of metaphor to tell you all that they mean. The unbelievers will say they are but words, but a slogan, but a flamboyant phrase. Every pedant, every demagogue, every cynic, every hypocrite, every troublemaker, and I am sorry to say, some others of an entirely different character, will try to downgrade them even to the extent of mockery and ridicule. But these are some of the things they do. They build your basic character. They mold you for your future roles as the custodians of the nation’s defense. They make you strong enough to know when you are weak, and brave enough to face yourself when you are afraid. They teach you to be proud and unbending in honest failure, but humble and gentle in success; not to substitute words for actions, not to seek the path of comfort, but to face the stress and spur of difficulty and challenge; to learn to stand up in the storm but to have compassion on those who fall; to master yourself before you seek to master others; to have a heart that is clean, a goal that is high; to learn to laugh, yet never forget how to weep; to reach into the future yet never neglect the past; to be serious yet never to take yourself too seriously; to be modest so that you will remember the simplicity of true greatness, the open mind of true wisdom, the meekness of true strength. They give you a temper of the will, a quality of the imagination, a vigor of the emotions, a freshness of the deep springs of life, a temperamental predominance of courage over timidity, of an appetite for adventure over love of ease. They create in your heart the sense of wonder, the unfailing hope of what next, and the joy and inspiration of life. They teach you in this way to be an officer and a gentleman. And what sort of soldiers are those you are to lead? Are they reliable? Are they brave? Are they capable of victory? Their story is known to all of you. It is the story of the American man-at-arms. My estimate of him was formed on the battlefield many, many years ago, and has never changed. I regarded him then as I regard him now — as one of the world’s noblest figures, not only as one of the finest military characters, but also as one of the most stainless. His name and fame are the birthright of every American citizen. In his youth and strength, his love and loyalty, he gave all that mortality can give. He needs no eulogy from me or from any other man. He has written his own history and written it in red on his enemy’s breast. But when I think of his patience under adversity, of his courage under fire, and of his modesty in victory, I am filled with an emotion of admiration I cannot put into words. He belongs to history as furnishing one of the greatest examples of successful patriotism. He belongs to posterity as the instructor of future generations in the principles of liberty and freedom. He belongs to the present, to us, by his virtues and by his achievements. In 20 campaigns, on a hundred battlefields, around a thousand campfires, I have witnessed that enduring fortitude, that patriotic self-abnegation, and that invincible determination which have carved his statue in the hearts of his people. From one end of the world to the other he has drained deep the chalice of courage. As I listened to those songs [of the glee club], in memory’s eye I could see those staggering columns of the First World War, bending under soggy packs, on many a weary march from dripping dusk to drizzling dawn, slogging ankle-deep through the mire of shell-shocked roads, to form grimly for the attack, blue-lipped, covered with sludge and mud, chilled by the wind and rain, driving home to their objective, and for many, to the judgment seat of God. I do not know the dignity of their birth, but I do know the glory of their death. They died unquestioning, uncomplaining, with faith in their hearts, and on their lips the hope that we would go on to victory. Always, for them: Duty, Honor, Country; always their blood and sweat and tears, as we sought the way and the light and the truth. And 20 years after, on the other side of the globe, again the filth of murky foxholes, the stench of ghostly trenches, the slime of dripping dugouts; those boiling suns of relentless heat, those torrential rains of devastating storms; the loneliness and utter desolation of jungle trails; the bitterness of long separation from those they loved and cherished; the deadly pestilence of tropical disease; the horror of stricken areas of war; their resolute and determined defense, their swift and sure attack, their indomitable purpose, their complete and decisive victory — always victory. Always through the bloody haze of their last reverberating shot, the vision of gaunt, ghastly men reverently following your password of: Duty, Honor, Country. The code which those words perpetuate embraces the highest moral laws and will stand the test of any ethics or philosophies ever promulgated for the uplift of mankind. Its requirements are for the things that are right, and its restraints are from the things that are wrong. The soldier, above all other men, is required to practice the greatest act of religious training — sacrifice. In battle and in the face of danger and death, he discloses those divine attributes which his Maker gave when he created man in his own image. No physical courage and no brute instinct can take the place of the Divine help which alone can sustain him. However horrible the incidents of war may be, the soldier who is called upon to offer and to give his life for his country is the noblest development of mankind. You now face a new world — a world of change. The thrust into outer space of the satellite, spheres, and missiles mark the beginning of another epoch in the long story of mankind. In the five or more billions of years the scientists tell us it has taken to form the earth, in the three or more billion years of development of the human race, there has never been a more abrupt or staggering evolution. We deal now not with things of this world alone, but with the illimitable distances and as yet unfathomed mysteries of the universe. We are reaching out for a new and boundless frontier. We speak in strange terms: of harnessing the cosmic energy; of making winds and tides work for us; of creating unheard synthetic materials to supplement or even replace our old standard basics; to purify sea water for our drink; of mining ocean floors for new fields of wealth and food; of disease preventatives to expand life into the hundreds of years; of controlling the weather for a more equitable distribution of heat and cold, of rain and shine; of space ships to the moon; of the primary target in war, no longer limited to the armed forces of an enemy, but instead to include his civil populations; of ultimate conflict between a united human race and the sinister forces of some other planetary galaxy; of such dreams and fantasies as to make life the most exciting of all time. And through all this welter of change and development, your mission remains fixed, determined, inviolable: it is to win our wars. Everything else in your professional career is but corollary to this vital dedication. All other public purposes, all other public projects, all other public needs, great or small, will find others for their accomplishment. But you are the ones who are trained to fight. Yours is the profession of arms, the will to win, the sure knowledge that in war there is no substitute for victory; that if you lose, the nation will be destroyed; that the very obsession of your public service must be: Duty, Honor, Country. Others will debate the controversial issues, national and international, which divide men’s minds; but serene, calm, aloof, you stand as the Nation’s war-guardian, as its lifeguard from the raging tides of international conflict, as its gladiator in the arena of battle. For a century and a half you have defended, guarded, and protected its hallowed traditions of liberty and freedom, of right and justice. Let civilian voices argue the merits or demerits of our processes of government; whether our strength is being sapped by deficit financing, indulged in too long, by federal paternalism grown too mighty, by power groups grown too arrogant, by politics grown too corrupt, by crime grown too rampant, by morals grown too low, by taxes grown too high, by extremists grown too violent; whether our personal liberties are as thorough and complete as they should be. These great national problems are not for your professional participation or military solution. Your guidepost stands out like a ten-fold beacon in the night: Duty, Honor, Country. You are the leaven which binds together the entire fabric of our national system of defense. From your ranks come the great captains who hold the nation’s destiny in their hands the moment the war tocsin sounds. The Long Gray Line has never failed us. Were you to do so, a million ghosts in olive drab, in brown khaki, in blue and gray, would rise from their white crosses thundering those magic words: Duty, Honor, Country. This does not mean that you are war mongers. On the contrary, the soldier, above all other people, prays for peace, for he must suffer and bear the deepest wounds and scars of war. But always in our ears ring the ominous words of Plato, that wisest of all philosophers: “Only the dead have seen the end of war.” The shadows are lengthening for me. The twilight is here. My days of old have vanished, tone and tint. They have gone glimmering through the dreams of things that were. Their memory is one of wondrous beauty, watered by tears, and coaxed and caressed by the smiles of yesterday. I listen vainly, but with thirsty ears, for the witching melody of faint bugles blowing reveille, of far drums beating the long roll. In my dreams I hear again the crash of guns, the rattle of musketry, the strange, mournful mutter of the battlefield. But in the evening of my memory, always I come back to West Point. Always there echoes and re-echoes: Duty, Honor, Country. Today marks my final roll call with you, but I want you to know that when I cross the river my last conscious thoughts will be of The Corps, and The Corps, and The Corps. I bid you farewell.',
 'kennedy': 'President Pitzer, Mr. Vice President, Governor, Congressman Thomas, Senator Wiley, and Congressman Miller, Mr. Webb, Mr. Bell, scientists, distinguished guests, and ladies and gentlemen: I appreciate your president having made me an honorary visiting professor, and I will assure you that my first lecture will be very brief. I am delighted to be here and I’m particularly delighted to be here on this occasion. We meet at a college noted for knowledge, in a city noted for progress, in a state noted for strength, and we stand in need of all three, for we meet in an hour of change and challenge, in a decade of hope and fear, in an age of both knowledge and ignorance. The greater our knowledge increases, the greater our ignorance unfolds. Despite the striking fact that most of the scientists that the world has ever known are alive and working today, despite the fact that this Nation’s own scientific manpower is doubling every 12 years in a rate of growth more than three times that of our population as a whole, despite that, the vast stretches of the unknown and the unanswered and the unfinished still far outstrip our collective comprehension. No man can fully grasp how far and how fast we have come, but condense, if you will, the 50,000 years of man’s recorded history in a time span of but a half-century. Stated in these terms, we know very little about the first 40 years, except at the end of them advanced man had learned to use the skins of animals to cover them. Then about 10 years ago, under this standard, man emerged from his caves to construct other kinds of shelter. Only five years ago man learned to write and use a cart with wheels. Christianity began less than two years ago. The printing press came this year, and then less than two months ago, during this whole 50-year span of human history, the steam engine provided a new source of power. Newton explored the meaning of gravity. Last month electric lights and telephones and automobiles and airplanes became available. Only last week did we develop penicillin and television and nuclear power, and now if America’s new spacecraft succeeds in reaching Venus, we will have literally reached the stars before midnight tonight. This is a breathtaking pace, and such a pace cannot help but create new ills as it dispels old, new ignorance, new problems, new dangers. Surely the opening vistas of space promise high costs and hardships, as well as high reward. So it is not surprising that some would have us stay where we are a little longer to rest, to wait. But this city of Houston, this state of Texas, this country of the United States was not built by those who waited and rested and wished to look behind them. This country was conquered by those who moved forward–and so will space. William Bradford, speaking in 1630 of the founding of the Plymouth Bay Colony, said that all great and honorable actions are accompanied with great difficulties, and both must be enterprised and overcome with answerable courage. If this capsule history of our progress teaches us anything, it is that man, in his quest for knowledge and progress, is determined and cannot be deterred. The exploration of space will go ahead, whether we join in it or not, and it is one of the great adventures of all time, and no nation which expects to be the leader of other nations can expect to stay behind in this race for space. Those who came before us made certain that this country rode the first waves of the industrial revolution, the first waves of modern invention, and the first wave of nuclear power, and this generation does not intend to founder in the backwash of the coming age of space. We mean to be a part of it–we mean to lead it. For the eyes of the world now look into space, to the moon and to the planets beyond, and we have vowed that we shall not see it governed by a hostile flag of conquest, but by a banner of freedom and peace. We have vowed that we shall not see space filled with weapons of mass destruction, but with instruments of knowledge and understanding. Yet the vows of this Nation can only be fulfilled if we in this Nation are first, and, therefore, we intend to be first. In short, our leadership in science and industry, our hopes for peace and security, our obligations to ourselves as well as others, all require us to make this effort, to solve these mysteries, to solve them for the good of all men, and to become the world’s leading space-faring nation. We set sail on this new sea because there is new knowledge to be gained, and new rights to be won, and they must be won and used for the progress of all people. For space science, like nuclear science and all technology, has no conscience of its own. Whether it will become a force for good or ill depends on man, and only if the United States occupies a position of pre-eminence can we help decide whether this new ocean will be a sea of peace or a new terrifying theater of war. I do not say that we should or will go unprotected against the hostile misuse of space any more than we go unprotected against the hostile use of land or sea, but I do say that space can be explored and mastered without feeding the fires of war, without repeating the mistakes that man has made in extending his writ around this globe of ours. There is no strife, no prejudice, no national conflict in outer space as yet. Its hazards are hostile to us all. Its conquest deserves the best of all mankind, and its opportunity for peaceful cooperation many never come again. But why, some say, the moon? Why choose this as our goal? And they may well ask why climb the highest mountain? Why, 35 years ago, fly the Atlantic? Why does Rice play Texas? We choose to go to the moon. We choose to go to the moon in this decade and do the other things, not because they are easy, but because they are hard, because that goal will serve to organize and measure the best of our energies and skills, because that challenge is one that we are willing to accept, one we are unwilling to postpone, and one which we intend to win, and the others, too. It is for these reasons that I regard the decision last year to shift our efforts in space from low to high gear as among the most important decisions that will be made during my incumbency in the office of the Presidency. In the last 24 hours we have seen facilities now being created for the greatest and most complex exploration in man’s history. We have felt the ground shake and the air shattered by the testing of a Saturn C-1 booster rocket, many times as powerful as the Atlas which launched John Glenn, generating power equivalent to 10,000 automobiles with their accelerators on the floor. We have seen the site where five F-1 rocket engines, each one as powerful as all eight engines of the Saturn combined, will be clustered together to make the advanced Saturn missile, assembled in a new building to be built at Cape Canaveral as tall as a 48 story structure, as wide as a city block, and as long as two lengths of this field. Within these last 19 months at least 45 satellites have circled the earth. Some 40 of them were made in the United States of America and they were far more sophisticated and supplied far more knowledge to the people of the world than those of the Soviet Union. The Mariner spacecraft now on its way to Venus is the most intricate instrument in the history of space science. The accuracy of that shot is comparable to firing a missile from Cape Canaveral and dropping it in this stadium between the 40-yard lines. Transit satellites are helping our ships at sea to steer a safer course. Tiros satellites have given us unprecedented warnings of hurricanes and storms, and will do the same for forest fires and icebergs. We have had our failures, but so have others, even if they do not admit them. And they may be less public. To be sure, we are behind, and will be behind for some time in manned flight. But we do not intend to stay behind, and in this decade, we shall make up and move ahead. The growth of our science and education will be enriched by new knowledge of our universe and environment, by new techniques of learning and mapping and observation, by new tools and computers for industry, medicine, the home as well as the school. Technical institutions, such as Rice, will reap the harvest of these gains. And finally, the space effort itself, while still in its infancy, has already created a great number of new companies, and tens of thousands of new jobs. Space and related industries are generating new demands in investment and skilled personnel, and this city and this state, and this region, will share greatly in this growth. What was once the furthest outpost on the old frontier of the West will be the furthest outpost on the new frontier of science and space. Houston, your city of Houston, with its Manned Spacecraft Center, will become the heart of a large scientific and engineering community. During the next 5 years the National Aeronautics and Space Administration expects to double the number of scientists and engineers in this area, to increase its outlays for salaries and expenses to $60 million a year; to invest some $200 million in plant and laboratory facilities; and to direct or contract for new space efforts over $1 billion from this center in this city. To be sure, all this costs us all a good deal of money. This year’s space budget is three times what it was in January 1961, and it is greater than the space budget of the previous eight years combined. That budget now stands at $5,400 million a year–a staggering sum, though somewhat less than we pay for cigarettes and cigars every year. Space expenditures will soon rise some more, from 40 cents per person per week to more than 50 cents a week for every man, woman and child in the United States, for we have given this program a high national priority–even though I realize that this is in some measure an act of faith and vision, for we do not now know what benefits await us. But if I were to say, my fellow citizens, that we shall send to the moon, 240,000 miles away from the control station in Houston, a giant rocket more than 300 feet tall, the length of this football field, made of new metal alloys, some of which have not yet been invented, capable of standing heat and stresses several times more than have ever been experienced, fitted together with a precision better than the finest watch, carrying all the equipment needed for propulsion, guidance, control, communications, food and survival, on an untried mission, to an unknown celestial body, and then return it safely to earth, re-entering the atmosphere at speeds of over 25,000 miles per hour, causing heat about half that of the temperature of the sun–almost as hot as it is here today–and do all this, and do it right, and do it first before this decade is out–then we must be bold. I’m the one who is doing all the work, so we just want you to stay cool for a minute. [laughter] However, I think we’re going to do it, and I think that we must pay what needs to be paid. I don’t think we ought to waste any money, but I think we ought to do the job. And this will be done in the decade of the Sixties. It may be done while some of you are still here at school at this college and university. It will be done during the terms of office of some of the people who sit here on this platform. But it will be done. And it will be done before the end of this decade. And I am delighted that this university is playing a part in putting a man on the moon as part of a great national effort of the United States of America. Many years ago the great British explorer George Mallory, who was to die on Mount Everest, was asked why did he want to climb it. He said, “Because it is there.” Well, space is there, and we’re going to climb it, and the moon and the planets are there, and new hopes for knowledge and peace are there. And, therefore, as we set sail we ask God’s blessing on the most hazardous and dangerous and greatest adventure on which man has ever embarked. Thank you.'}


# Function to process the text
def processa_texto(text):
    
    # Convert to lower case
    doc = nlp(text.lower())
    
    # List to store result
    result = []
    
    # Loop
    for token in doc:
        if token.text in nlp.Defaults.stop_words:
            continue
        if token.is_punct:
            continue
        if token.lemma_ == '-PRON-':
            continue
        result.append(token.lemma_)
    return " ".join(result)


# View
df_audio_videos


# Iterate through the speeches and compare the transcriptions of Amazon vs SpaCy
for index, row in df_audio_videos.iterrows():
    
    print(index, row)
    
    # Load the dictionary
    nlp = spacy.load('en_core_web_lg') 
    
    # Process the original transcription text
    original_transcription = nlp(processa_texto(original_transcriptions.get(index)))
    
    # Process the text from Amazon Transcribe
    transcribe_transcription = nlp(processa_texto(get_texto(bucket_name, row.json_transcription)))
    
    # Calculate similarity
    similaridade = original_transcription.similarity(transcribe_transcription)
    
    # Save result to a DataFrame
    df_audio_videos.at[index, 'similaridade'] = similaridade
    
    print(f'\nSimilaridade Word2vec para o discurso de {index}: {similaridade}')

luther_king youtube_urls                https://www.youtube.com/watch?v=I47Y6VHc3Ms
filename                                                       king.mp3
uri                            s3://dsa-sistemas-mini-projeto5/king.mp3
transcription_url     https://dsa-sistemas-mini-projeto5.s3.amazonaw...
json_transcription            luther_king_speech_dsa-mini-projeto5.json
Name: luther_king, dtype: object

Similaridade Word2vec para o discurso de luther_king: 0.9980868560317497
kennedy youtube_urls                https://www.youtube.com/watch?v=QAmHcdwKgtQ
filename                                                    kennedy.mp3
uri                         s3://dsa-sistemas-mini-projeto5/kennedy.mp3
transcription_url     https://dsa-sistemas-mini-projeto5.s3.amazonaw...
json_transcription                kennedy_speech_dsa-mini-projeto5.json
Name: kennedy, dtype: object

Similaridade Word2vec para o discurso de kennedy: 0.9995996536888027
churchill youtube_urls                https://www.youtube.com/watch?v=s_LncVnecLA
filename                                                  churchill.mp3
uri                       s3://dsa-sistemas-mini-projeto5/churchill.mp3
transcription_url     https://dsa-sistemas-mini-projeto5.s3.amazonaw...
json_transcription              churchill_speech_dsa-mini-projeto5.json
Name: churchill, dtype: object

Similaridade Word2vec para o discurso de churchill: 0.9975913181060134
reagan youtube_urls           https://www.youtube.com/watch?v=5MDFX-dNtsM&t=6s
filename                                                     reagan.mp3
uri                          s3://dsa-sistemas-mini-projeto5/reagan.mp3
transcription_url     https://dsa-sistemas-mini-projeto5.s3.amazonaw...
json_transcription                 reagan_speech_dsa-mini-projeto5.json
Name: reagan, dtype: object

Similaridade Word2vec para o discurso de reagan: 0.9994992064962662
macarthur youtube_urls          https://www.youtube.com/watch?v=_42_aLGkRpg&t=19s
filename                                                  macarthur.mp3
uri                       s3://dsa-sistemas-mini-projeto5/macarthur.mp3
transcription_url     https://dsa-sistemas-mini-projeto5.s3.amazonaw...
json_transcription              macarthur_speech_dsa-mini-projeto5.json
Name: macarthur, dtype: object

Similaridade Word2vec para o discurso de macarthur: 0.9990477621523232


# Label values
y_labels = [label.replace('_', ' ').capitalize() for label in df_audio_videos.index]
print(y_labels)

['Luther king', 'Kennedy', 'Churchill', 'Reagan', 'Macarthur']


# Ser similarity values to percentage
x_values = [x * 100 for x in df_audio_videos.similaridade ]
print(x_values)

[99.80868560317498, 99.95996536888026, 99.75913181060135, 99.94992064962662, 99.90477621523232]


# Save to DataFrame
df_audio_videos['label'], df_audio_videos['similaridade_percentual'] = y_labels, x_values


# View
df_audio_videos


# Topic modeling
for index, row in df_audio_videos.iterrows():
    
    print(f"Renderizando o texto de {index}")    
    
    # Load the dictionary
    nlp = spacy.load('en_core_web_lg')
    
    # SpaCy transcription
    original_transcription = nlp(original_transcriptions.get(index))    
    
    # Amazon Transcribe transcription
    transcribe_transcription = nlp(get_texto(bucket_name, row.json_transcription))
    
    # Render 
    svg_original = spacy.displacy.render(original_transcription, style = "ent", jupyter = False)
    svg_transcribe = spacy.displacy.render(transcribe_transcription, style = "ent", jupyter = False)
    
    # Grava em disco
    with open(f'dados/{index}_original.html', 'w',  encoding = "utf-8") as page:
        page.write(svg_original)
    with open(f'dados/{index}_transcribe.html', 'w',  encoding = "utf-8") as page:
        page.write(svg_transcribe)

Renderizando o texto de luther_king
Renderizando o texto de kennedy
Renderizando o texto de churchill
Renderizando o texto de reagan
Renderizando o texto de macarthur


# Function to analyze sentiment with Amazon Comprehend
def analisa_sentimento(text):
    
    # List
    list_parts = []
    
    # Text for analysis
    text_for_analysis = ''
    
    # Loop through sentences
    for sentence in text.split('.'):
        current_text = text_for_analysis + f'{sentence}.'

        if len(current_text.encode('utf-8')) > 5000:
            list_parts.append([len(text_for_analysis), text_for_analysis])
            text_for_analysis = f'{sentence}.'
        else:
            text_for_analysis += f'{sentence}.'

    # Append
    list_parts.append([len(text_for_analysis), text_for_analysis])
    
    # Dictionary
    dict_comprehend = {}
    
    # Loop
    for t_parts in list_parts:
        
        # Connect to Amazon Comprehend
        comprehend_client = boto3.client(service_name = 'comprehend', region_name = 'us-east-2')
        
        # Detect sentiment on the text
        sentimentData = comprehend_client.detect_sentiment(Text = t_parts[1], LanguageCode = 'en')
        
        # Add to dictionary
        dict_comprehend[t_parts[0]] = sentimentData
        dict_comprehend[t_parts[0]]['ratio'] = t_parts[0]/float(len(text))

    final_dict = {'Positive':0, 'Negative':0, 'Neutral':0, 'Mixed':0}
    list_sentiments = ['Positive', 'Negative', 'Neutral', 'Mixed']
    
    for sentiment in list_sentiments:
        for key, value in dict_comprehend.items():
            final_dict[sentiment] += value.get('SentimentScore').get(sentiment) * value.get('ratio')
    
    return final_dict


# Run the sentiment analysis
for index, row in df_audio_videos.iterrows():
    
    # Obtain the text from Amazon Transcribe
    transcript_text = get_texto(bucket_name, row.json_transcription)
    
    # Run the sentiment analysis
    comprehend_results = analisa_sentimento(transcript_text)
    
    # Add the result to the DataFrame
    for k,v in comprehend_results.items():
        df_audio_videos.at[index, k] = v


# Round the values to percentages
df_audio_videos = df_audio_videos.round(2)


# View
df_audio_videos


# DataFrame with labels and detected sentiments
df_sentimento = df_audio_videos[['label', 'Positive', 'Negative', 'Neutral', 'Mixed']]


# View
df_sentimento


# Melt the DataFrame to simplify visualization
df_sentimento_melt = pd.melt(df_sentimento, id_vars = "label", var_name = "sentimento", value_name = "sentiment_rate")


# View
df_sentimento_melt


# Adjust percentages
df_sentimento_melt['sentiment_percentage'] = df_sentimento_melt.sentiment_rate * 100


# View
df_sentimento_melt


# Plot
sns_plot = sns.factorplot(x = 'label', 
                          y = 'sentiment_percentage', 
                          hue = 'sentimento', 
                          data = df_sentimento_melt, 
                          kind = 'bar')
sns_plot.set_xticklabels(rotation = 16)
sns_plot.set_xlabels('Discurso', fontsize = 13)
sns_plot.set_ylabels('% Sentimento', fontsize = 13)
sns_plot.set_titles('Análise de Sentimento')
sns_plot.savefig("imagens/analise.png", orientation = 'landscape', bbox_inches = 'tight')


# Reindex by sentiment
df_sentimento_reindexado = df_sentimento.drop('label', axis = 1).T
df_sentimento_reindexado = df_sentimento_reindexado.reindex(index = ['Positive', 'Negative', 'Neutral', 'Mixed'])


# View
df_sentimento_reindexado


# Radar Plot

# Categories
categories = ['Positive', 'Negative', 'Neutral', 'Mixed']
colors = ['']
N = len(categories)
legend = []

# Loop
for n in range(len(df_sentimento_reindexado.columns)):
    
    # Values
    values = df_sentimento_reindexado[df_sentimento_reindexado.columns[n]].tolist()
    values += values[:1]
    legend.append(df_sentimento_reindexado.columns[n])

    # Angles
    angles = [n / float(N) * 2 * pi for n in range(N)]
    angles += angles[:1]
    
    # Subplot
    ax = plt.subplot(111, polar = True)
    plt.xticks(angles[:-1], categories, color = 'black', size = 8)

    # Labels
    ax.set_rlabel_position(0)
    plt.yticks([0.2,0.4,0.6, 0.8], ["20", "40", "60", ""], color = "black", size = 7)
    plt.ylim(0,0.80)
    ax.plot(angles, values, linewidth = 1, linestyle = 'solid')
    
    # legend
    plt.legend(legend, bbox_to_anchor = (1.04,1), loc = "upper left")

    # Fill
    ax.fill(angles, values, 'b', alpha = 0.05)

plt.savefig(f'imagens/radar_plot.png', orientation = 'landscape', bbox_inches = 'tight')
plt.show()

	label	Positive	Negative	Neutral	Mixed
luther_king	Luther king	0.54	0.17	0.15	0.13
kennedy	Kennedy	0.46	0.03	0.18	0.33
churchill	Churchill	0.10	0.25	0.51	0.15
reagan	Reagan	0.43	0.06	0.28	0.23
macarthur	Macarthur	0.65	0.02	0.26	0.07

	luther_king	kennedy	churchill	reagan	macarthur
Positive	0.54	0.46	0.10	0.43	0.65
Negative	0.17	0.03	0.25	0.06	0.02
Neutral	0.15	0.18	0.51	0.28	0.26
Mixed	0.13	0.33	0.15	0.23	0.07

Youtube Sentiment Analysis with Amazon Comprehend¶

Installing and Loading Packages¶

Configurating the AWS Client¶

Extracting Audio and Text from Youtube Videos¶

Downloading and Preparing Data¶

Storing Videos on Amazon S3¶

Transcribing the Text in the Audio Files with Amazon Transcribe¶

Comparing the Amazon Comprehend Transcription with the SpaCy Transcription¶

Topic Modeling and Named Entity Recognition¶

Sentiment Analysis with Amazon Comprehend¶

Visualize the Results from Sentiment Analysis¶

End¶

	youtube_urls
luther_king	https://www.youtube.com/watch?v=I47Y6VHc3Ms
kennedy	https://www.youtube.com/watch?v=QAmHcdwKgtQ
churchill	https://www.youtube.com/watch?v=s_LncVnecLA
reagan	https://www.youtube.com/watch?v=5MDFX-dNtsM&t=6s
macarthur	https://www.youtube.com/watch?v=_42_aLGkRpg&t=19s

	youtube_urls	filename
luther_king	https://www.youtube.com/watch?v=I47Y6VHc3Ms	king.mp3
kennedy	https://www.youtube.com/watch?v=QAmHcdwKgtQ	kennedy.mp3
churchill	https://www.youtube.com/watch?v=s_LncVnecLA	churchill.mp3
reagan	https://www.youtube.com/watch?v=5MDFX-dNtsM&t=6s	reagan.mp3
macarthur	https://www.youtube.com/watch?v=_42_aLGkRpg&t=19s	macarthur.mp3