NYT API#
To use the New York Times API, register for an API key here.
For more detailed explanation on using the API, see this resource: https://nicksubic.medium.com/a-guide-to-querying-the-new-york-times-api-with-python-b621556236f8
import requests
from time import sleep
creating API calls with keys#
# key
key = 'HHhPAjnfEF2EmLtf9PdGsLEQZizR3iQu'
# query
query = 'migrant'
# base URL
url = f'https://api.nytimes.com/svc/search/v2/articlesearch.json?&q={query}&api-key={key}'
response = requests.get(url)
response
<Response [200]>
type(response)
requests.models.Response
dir(response)
['__attrs__',
'__bool__',
'__class__',
'__delattr__',
'__dict__',
'__dir__',
'__doc__',
'__enter__',
'__eq__',
'__exit__',
'__format__',
'__ge__',
'__getattribute__',
'__getstate__',
'__gt__',
'__hash__',
'__init__',
'__init_subclass__',
'__iter__',
'__le__',
'__lt__',
'__module__',
'__ne__',
'__new__',
'__nonzero__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__setattr__',
'__setstate__',
'__sizeof__',
'__str__',
'__subclasshook__',
'__weakref__',
'_content',
'_content_consumed',
'_next',
'apparent_encoding',
'close',
'connection',
'content',
'cookies',
'elapsed',
'encoding',
'headers',
'history',
'is_permanent_redirect',
'is_redirect',
'iter_content',
'iter_lines',
'json',
'links',
'next',
'ok',
'raise_for_status',
'raw',
'reason',
'request',
'status_code',
'text',
'url']
parsing our data#
We want to turn our response object into a dict data structure, which we can more easily explore using Python.
parsed = response.json()
parsed.keys()
dict_keys(['status', 'copyright', 'response'])
Use methods like keys() to see what is contained within the parsed object.
parsed['response'].keys()
dict_keys(['docs', 'metadata'])
Use brackets to go deeper into the dict structure. First, check out the meta key, then the docs key.
type(parsed['response']['meta'])
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In[10], line 1
----> 1 type(parsed['response']['meta'])
KeyError: 'meta'
parsed['response']['meta']
{'hits': 32616, 'offset': 0, 'time': 82}
Inside docs, we have a list.
type(parsed['response']['docs'])
list
The data in docs is absolutely massive, so we will just print out the first item in that object.
parsed['response']['docs'][0]
{'abstract': 'There has been a dramatic drop in the number of people gathering at the U.S. border and trying to cross. Can it help Mexico stave off President Trump’s threatened tariffs?',
'web_url': 'https://www.nytimes.com/2025/03/03/world/americas/mexico-border-migration-trump-tariffs-deadline.html',
'snippet': 'There has been a dramatic drop in the number of people gathering at the U.S. border and trying to cross. Can it help Mexico stave off President Trump’s threatened tariffs?',
'lead_paragraph': 'On the eve of President Trump’s deadline to impose tariffs on Mexico, one thing is hard to miss on the Mexican side of the border: The migrants are gone.',
'print_section': 'A',
'print_page': '4',
'source': 'The New York Times',
'multimedia': [{'rank': 0,
'subtype': 'xlarge',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-articleLarge.jpg',
'height': 400,
'width': 600,
'legacy': {'xlarge': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-articleLarge.jpg',
'xlargewidth': 600,
'xlargeheight': 400},
'subType': 'xlarge',
'crop_name': 'articleLarge'},
{'rank': 0,
'subtype': 'popup',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-popup.jpg',
'height': 433,
'width': 650,
'legacy': {},
'subType': 'popup',
'crop_name': 'popup'},
{'rank': 0,
'subtype': 'blog480',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blog480.jpg',
'height': 320,
'width': 480,
'legacy': {},
'subType': 'blog480',
'crop_name': 'blog480'},
{'rank': 0,
'subtype': 'blog533',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blog533.jpg',
'height': 355,
'width': 533,
'legacy': {},
'subType': 'blog533',
'crop_name': 'blog533'},
{'rank': 0,
'subtype': 'blog427',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blog427.jpg',
'height': 285,
'width': 427,
'legacy': {},
'subType': 'blog427',
'crop_name': 'blog427'},
{'rank': 0,
'subtype': 'tmagSF',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-tmagSF.jpg',
'height': 241,
'width': 362,
'legacy': {},
'subType': 'tmagSF',
'crop_name': 'tmagSF'},
{'rank': 0,
'subtype': 'tmagArticle',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-tmagArticle.jpg',
'height': 395,
'width': 592,
'legacy': {},
'subType': 'tmagArticle',
'crop_name': 'tmagArticle'},
{'rank': 0,
'subtype': 'slide',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-slide.jpg',
'height': 400,
'width': 600,
'legacy': {},
'subType': 'slide',
'crop_name': 'slide'},
{'rank': 0,
'subtype': 'jumbo',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-jumbo.jpg',
'height': 683,
'width': 1024,
'legacy': {},
'subType': 'jumbo',
'crop_name': 'jumbo'},
{'rank': 0,
'subtype': 'superJumbo',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-superJumbo.jpg',
'height': 1365,
'width': 2048,
'legacy': {},
'subType': 'superJumbo',
'crop_name': 'superJumbo'},
{'rank': 0,
'subtype': 'blog225',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blog225.jpg',
'height': 150,
'width': 225,
'legacy': {},
'subType': 'blog225',
'crop_name': 'blog225'},
{'rank': 0,
'subtype': 'master1050',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master1050.jpg',
'height': 700,
'width': 1050,
'legacy': {},
'subType': 'master1050',
'crop_name': 'master1050'},
{'rank': 0,
'subtype': 'master675',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master675.jpg',
'height': 450,
'width': 675,
'legacy': {},
'subType': 'master675',
'crop_name': 'master675'},
{'rank': 0,
'subtype': 'master495',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master495.jpg',
'height': 330,
'width': 495,
'legacy': {},
'subType': 'master495',
'crop_name': 'master495'},
{'rank': 0,
'subtype': 'master180',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master180.jpg',
'height': 120,
'width': 180,
'legacy': {},
'subType': 'master180',
'crop_name': 'master180'},
{'rank': 0,
'subtype': 'master315',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master315.jpg',
'height': 210,
'width': 315,
'legacy': {},
'subType': 'master315',
'crop_name': 'master315'},
{'rank': 0,
'subtype': 'master768',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-master768.jpg',
'height': 512,
'width': 768,
'legacy': {},
'subType': 'master768',
'crop_name': 'master768'},
{'rank': 0,
'subtype': 'thumbnail',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-thumbStandard.jpg',
'height': 75,
'width': 75,
'legacy': {'thumbnail': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-thumbStandard.jpg',
'thumbnailwidth': 75,
'thumbnailheight': 75},
'subType': 'thumbnail',
'crop_name': 'thumbStandard'},
{'rank': 0,
'subtype': 'blogSmallThumb',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blogSmallThumb.jpg',
'height': 50,
'width': 50,
'legacy': {},
'subType': 'blogSmallThumb',
'crop_name': 'blogSmallThumb'},
{'rank': 0,
'subtype': 'thumbLarge',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-thumbLarge.jpg',
'height': 150,
'width': 150,
'legacy': {},
'subType': 'thumbLarge',
'crop_name': 'thumbLarge'},
{'rank': 0,
'subtype': 'smallSquare168',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-smallSquare168.jpg',
'height': 168,
'width': 168,
'legacy': {},
'subType': 'smallSquare168',
'crop_name': 'smallSquare168'},
{'rank': 0,
'subtype': 'smallSquare252',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-smallSquare252.jpg',
'height': 252,
'width': 252,
'legacy': {},
'subType': 'smallSquare252',
'crop_name': 'smallSquare252'},
{'rank': 0,
'subtype': 'square320',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-square320.jpg',
'height': 320,
'width': 320,
'legacy': {},
'subType': 'square320',
'crop_name': 'square320'},
{'rank': 0,
'subtype': 'moth',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-moth.jpg',
'height': 151,
'width': 151,
'legacy': {},
'subType': 'moth',
'crop_name': 'moth'},
{'rank': 0,
'subtype': 'filmstrip',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-filmstrip.jpg',
'height': 190,
'width': 190,
'legacy': {},
'subType': 'filmstrip',
'crop_name': 'filmstrip'},
{'rank': 0,
'subtype': 'square640',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-square640.jpg',
'height': 640,
'width': 640,
'legacy': {},
'subType': 'square640',
'crop_name': 'square640'},
{'rank': 0,
'subtype': 'mediumSquare149',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumSquare149.jpg',
'height': 149,
'width': 149,
'legacy': {},
'subType': 'mediumSquare149',
'crop_name': 'mediumSquare149'},
{'rank': 0,
'subtype': 'mediumSquareAt3X',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumSquareAt3X.jpg',
'height': 1800,
'width': 1800,
'legacy': {},
'subType': 'mediumSquareAt3X',
'crop_name': 'mediumSquareAt3X'},
{'rank': 0,
'subtype': 'sfSpan',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-sfSpan.jpg',
'height': 263,
'width': 395,
'legacy': {},
'subType': 'sfSpan',
'crop_name': 'sfSpan'},
{'rank': 0,
'subtype': 'largeHorizontal375',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-largeHorizontal375.jpg',
'height': 250,
'width': 375,
'legacy': {},
'subType': 'largeHorizontal375',
'crop_name': 'largeHorizontal375'},
{'rank': 0,
'subtype': 'largeHorizontalJumbo',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-largeHorizontalJumbo.jpg',
'height': 683,
'width': 1024,
'legacy': {},
'subType': 'largeHorizontalJumbo',
'crop_name': 'largeHorizontalJumbo'},
{'rank': 0,
'subtype': 'horizontalMediumAt2X',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-horizontalMediumAt2X.jpg',
'height': 3333,
'width': 5000,
'legacy': {},
'subType': 'horizontalMediumAt2X',
'crop_name': 'horizontalMediumAt2X'},
{'rank': 0,
'subtype': 'hpLarge',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-hpLarge.jpg',
'height': 287,
'width': 511,
'legacy': {},
'subType': 'hpLarge',
'crop_name': 'hpLarge'},
{'rank': 0,
'subtype': 'largeWidescreen573',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-largeWidescreen573.jpg',
'height': 322,
'width': 573,
'legacy': {},
'subType': 'largeWidescreen573',
'crop_name': 'largeWidescreen573'},
{'rank': 0,
'subtype': 'largeWidescreen1050',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-largeWidescreen1050.jpg',
'height': 591,
'width': 1050,
'legacy': {},
'subType': 'largeWidescreen1050',
'crop_name': 'largeWidescreen1050'},
{'rank': 0,
'subtype': 'wide',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-thumbWide.jpg',
'height': 126,
'width': 190,
'legacy': {'widewidth': 190,
'wideheight': 126,
'wide': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-thumbWide.jpg'},
'subType': 'wide',
'crop_name': 'thumbWide'},
{'rank': 0,
'subtype': 'videoThumb',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoThumb.jpg',
'height': 50,
'width': 75,
'legacy': {},
'subType': 'videoThumb',
'crop_name': 'videoThumb'},
{'rank': 0,
'subtype': 'videoLarge',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoLarge.jpg',
'height': 507,
'width': 768,
'legacy': {},
'subType': 'videoLarge',
'crop_name': 'videoLarge'},
{'rank': 0,
'subtype': 'mediumThreeByTwo210',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumThreeByTwo210.jpg',
'height': 140,
'width': 210,
'legacy': {},
'subType': 'mediumThreeByTwo210',
'crop_name': 'mediumThreeByTwo210'},
{'rank': 0,
'subtype': 'mediumThreeByTwo225',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumThreeByTwo225.jpg',
'height': 150,
'width': 225,
'legacy': {},
'subType': 'mediumThreeByTwo225',
'crop_name': 'mediumThreeByTwo225'},
{'rank': 0,
'subtype': 'mediumThreeByTwo440',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumThreeByTwo440.jpg',
'height': 293,
'width': 440,
'legacy': {},
'subType': 'mediumThreeByTwo440',
'crop_name': 'mediumThreeByTwo440'},
{'rank': 0,
'subtype': 'mediumThreeByTwo252',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumThreeByTwo252.jpg',
'height': 168,
'width': 252,
'legacy': {},
'subType': 'mediumThreeByTwo252',
'crop_name': 'mediumThreeByTwo252'},
{'rank': 0,
'subtype': 'mediumThreeByTwo378',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumThreeByTwo378.jpg',
'height': 252,
'width': 378,
'legacy': {},
'subType': 'mediumThreeByTwo378',
'crop_name': 'mediumThreeByTwo378'},
{'rank': 0,
'subtype': 'threeByTwoLargeAt2X',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-threeByTwoLargeAt2X.jpg',
'height': 3333,
'width': 5000,
'legacy': {},
'subType': 'threeByTwoLargeAt2X',
'crop_name': 'threeByTwoLargeAt2X'},
{'rank': 0,
'subtype': 'threeByTwoMediumAt2X',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-threeByTwoMediumAt2X.jpg',
'height': 1000,
'width': 1500,
'legacy': {},
'subType': 'threeByTwoMediumAt2X',
'crop_name': 'threeByTwoMediumAt2X'},
{'rank': 0,
'subtype': 'threeByTwoSmallAt2X',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-threeByTwoSmallAt2X.jpg',
'height': 400,
'width': 600,
'legacy': {},
'subType': 'threeByTwoSmallAt2X',
'crop_name': 'threeByTwoSmallAt2X'},
{'rank': 0,
'subtype': 'articleInline',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-articleInline.jpg',
'height': 127,
'width': 190,
'legacy': {},
'subType': 'articleInline',
'crop_name': 'articleInline'},
{'rank': 0,
'subtype': 'hpSmall',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-hpSmall.jpg',
'height': 109,
'width': 163,
'legacy': {},
'subType': 'hpSmall',
'crop_name': 'hpSmall'},
{'rank': 0,
'subtype': 'blogSmallInline',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-blogSmallInline.jpg',
'height': 101,
'width': 151,
'legacy': {},
'subType': 'blogSmallInline',
'crop_name': 'blogSmallInline'},
{'rank': 0,
'subtype': 'mediumFlexible177',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mediumFlexible177.jpg',
'height': 118,
'width': 177,
'legacy': {},
'subType': 'mediumFlexible177',
'crop_name': 'mediumFlexible177'},
{'rank': 0,
'subtype': 'videoSmall',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSmall.jpg',
'height': 281,
'width': 500,
'legacy': {},
'subType': 'videoSmall',
'crop_name': 'videoSmall'},
{'rank': 0,
'subtype': 'videoHpMedium',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoHpMedium.jpg',
'height': 211,
'width': 375,
'legacy': {},
'subType': 'videoHpMedium',
'crop_name': 'videoHpMedium'},
{'rank': 0,
'subtype': 'videoSixteenByNine600',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine600.jpg',
'height': 338,
'width': 600,
'legacy': {},
'subType': 'videoSixteenByNine600',
'crop_name': 'videoSixteenByNine600'},
{'rank': 0,
'subtype': 'videoSixteenByNine540',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine540.jpg',
'height': 304,
'width': 540,
'legacy': {},
'subType': 'videoSixteenByNine540',
'crop_name': 'videoSixteenByNine540'},
{'rank': 0,
'subtype': 'videoSixteenByNine495',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine495.jpg',
'height': 278,
'width': 495,
'legacy': {},
'subType': 'videoSixteenByNine495',
'crop_name': 'videoSixteenByNine495'},
{'rank': 0,
'subtype': 'videoSixteenByNine390',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine390.jpg',
'height': 219,
'width': 390,
'legacy': {},
'subType': 'videoSixteenByNine390',
'crop_name': 'videoSixteenByNine390'},
{'rank': 0,
'subtype': 'videoSixteenByNine1050',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine1050.jpg',
'height': 591,
'width': 1050,
'legacy': {},
'subType': 'videoSixteenByNine1050',
'crop_name': 'videoSixteenByNine1050'},
{'rank': 0,
'subtype': 'videoSixteenByNine480',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine480.jpg',
'height': 270,
'width': 480,
'legacy': {},
'subType': 'videoSixteenByNine480',
'crop_name': 'videoSixteenByNine480'},
{'rank': 0,
'subtype': 'videoSixteenByNine310',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine310.jpg',
'height': 174,
'width': 310,
'legacy': {},
'subType': 'videoSixteenByNine310',
'crop_name': 'videoSixteenByNine310'},
{'rank': 0,
'subtype': 'videoSixteenByNine225',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine225.jpg',
'height': 126,
'width': 225,
'legacy': {},
'subType': 'videoSixteenByNine225',
'crop_name': 'videoSixteenByNine225'},
{'rank': 0,
'subtype': 'videoSixteenByNine96',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine96.jpg',
'height': 54,
'width': 96,
'legacy': {},
'subType': 'videoSixteenByNine96',
'crop_name': 'videoSixteenByNine96'},
{'rank': 0,
'subtype': 'videoSixteenByNine3000',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine3000.jpg',
'height': 1687,
'width': 3000,
'legacy': {},
'subType': 'videoSixteenByNine3000',
'crop_name': 'videoSixteenByNine3000'},
{'rank': 0,
'subtype': 'videoSixteenByNine768',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine768.jpg',
'height': 432,
'width': 768,
'legacy': {},
'subType': 'videoSixteenByNine768',
'crop_name': 'videoSixteenByNine768'},
{'rank': 0,
'subtype': 'videoSixteenByNine150',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNine150.jpg',
'height': 84,
'width': 150,
'legacy': {},
'subType': 'videoSixteenByNine150',
'crop_name': 'videoSixteenByNine150'},
{'rank': 0,
'subtype': 'videoSixteenByNineJumbo1600',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoSixteenByNineJumbo1600.jpg',
'height': 900,
'width': 1600,
'legacy': {},
'subType': 'videoSixteenByNineJumbo1600',
'crop_name': 'videoSixteenByNineJumbo1600'},
{'rank': 0,
'subtype': 'miniMoth',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-miniMoth.jpg',
'height': 70,
'width': 151,
'legacy': {},
'subType': 'miniMoth',
'crop_name': 'miniMoth'},
{'rank': 0,
'subtype': 'windowsTile336H',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-windowsTile336H.jpg',
'height': 336,
'width': 694,
'legacy': {},
'subType': 'windowsTile336H',
'crop_name': 'windowsTile336H'},
{'rank': 0,
'subtype': 'videoFifteenBySeven1305',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoFifteenBySeven1305.jpg',
'height': 609,
'width': 1305,
'legacy': {},
'subType': 'videoFifteenBySeven1305',
'crop_name': 'videoFifteenBySeven1305'},
{'rank': 0,
'subtype': 'videoFifteenBySeven2610',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-videoFifteenBySeven2610.jpg',
'height': 1218,
'width': 2610,
'legacy': {},
'subType': 'videoFifteenBySeven2610',
'crop_name': 'videoFifteenBySeven2610'},
{'rank': 0,
'subtype': 'facebookJumbo',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-facebookJumbo.jpg',
'height': 550,
'width': 1050,
'legacy': {},
'subType': 'facebookJumbo',
'crop_name': 'facebookJumbo'},
{'rank': 0,
'subtype': 'watch308',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-watch308.jpg',
'height': 348,
'width': 312,
'legacy': {},
'subType': 'watch308',
'crop_name': 'watch308'},
{'rank': 0,
'subtype': 'watch268',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-watch268.jpg',
'height': 303,
'width': 272,
'legacy': {},
'subType': 'watch268',
'crop_name': 'watch268'},
{'rank': 0,
'subtype': 'verticalTwoByThree735',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-verticalTwoByThree735.jpg',
'height': 1103,
'width': 735,
'legacy': {},
'subType': 'verticalTwoByThree735',
'crop_name': 'verticalTwoByThree735'},
{'rank': 0,
'subtype': 'mobileMasterAt3x',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-mobileMasterAt3x.jpg',
'height': 1800,
'width': 1800,
'legacy': {},
'subType': 'mobileMasterAt3x',
'crop_name': 'mobileMasterAt3x'},
{'rank': 0,
'subtype': 'googleFourByThree',
'caption': None,
'credit': None,
'type': 'image',
'url': 'images/2025/03/03/multimedia/03mexico-migration01-photo-tlbq/03mexico-migration01-photo-tlbq-googleFourByThree.jpg',
'height': 600,
'width': 800,
'legacy': {},
'subType': 'googleFourByThree',
'crop_name': 'googleFourByThree'}],
'headline': {'main': 'On Mexico’s Once-Packed Border, Few Migrants Remain',
'kicker': None,
'content_kicker': None,
'print_headline': 'On a Once-Packed Border, Just a Few Migrants Remain',
'name': None,
'seo': None,
'sub': None},
'keywords': [{'name': 'subject',
'value': 'United States International Relations',
'rank': 1,
'major': 'N'},
{'name': 'subject', 'value': 'Illegal Immigration', 'rank': 2, 'major': 'N'},
{'name': 'subject', 'value': 'Asylum, Right of', 'rank': 3, 'major': 'N'},
{'name': 'subject', 'value': 'Customs (Tariff)', 'rank': 4, 'major': 'N'},
{'name': 'subject',
'value': 'Politics and Government',
'rank': 5,
'major': 'N'},
{'name': 'glocations', 'value': 'Mexico', 'rank': 6, 'major': 'N'},
{'name': 'persons', 'value': 'Trump, Donald J', 'rank': 7, 'major': 'N'},
{'name': 'glocations', 'value': 'Tijuana (Mexico)', 'rank': 8, 'major': 'N'},
{'name': 'glocations',
'value': 'Ciudad Juarez (Mexico)',
'rank': 9,
'major': 'N'},
{'name': 'glocations', 'value': 'Juarez (Mexico)', 'rank': 10, 'major': 'N'},
{'name': 'glocations',
'value': 'Matamoros (Mexico)',
'rank': 11,
'major': 'N'},
{'name': 'organizations',
'value': 'Customs and Border Protection (US)',
'rank': 12,
'major': 'N'}],
'pub_date': '2025-03-03T14:00:34+0000',
'document_type': 'article',
'news_desk': 'Foreign',
'section_name': 'World',
'subsection_name': 'Americas',
'byline': {'original': 'By Annie Correal and Alejandro Cegarra',
'person': [{'firstname': 'Annie',
'middlename': None,
'lastname': 'Correal',
'qualifier': None,
'title': None,
'role': 'reported',
'organization': '',
'rank': 1},
{'firstname': 'Alejandro',
'middlename': None,
'lastname': 'Cegarra',
'qualifier': None,
'title': None,
'role': 'reported',
'organization': '',
'rank': 2}],
'organization': None},
'type_of_material': 'News',
'_id': 'nyt://article/5a74da49-82a5-5147-ae1f-58f9c66437f0',
'word_count': 1595,
'uri': 'nyt://article/5a74da49-82a5-5147-ae1f-58f9c66437f0'}
The very first item in docs appears to be information about a single article.
parsed['response']['docs'][0].keys()
dict_keys(['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'subsection_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri'])
Let’s save the information in docs to a list. That way, we will have an easier time going through the data, and won’t have to re-type out all of the keys that our data is nested within.
articles = parsed['response']['docs']
We have a list of articles. Let’s look at the first one.
type(articles)
list
articles[0].keys()
dict_keys(['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'subsection_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri'])
paginating through results#
We have 10 articles, because the NYTimes API only allows us to get 10 results at a time. We can paginate through the results by adding an &page= parameter, combined with a sleep() function, which allows us to insert pauses in our request (as to not overload the NYTimes servers).
len(articles)
10
We will write a loop that does the following:
create a url with variables for pagination, query, and our key
make an API call, save our response to
responseparse our data from the
responseaccess the response data we want in
docssave it to a list
results = []
query = 'migrant'
for i in range(0, 5):
url = f'https://api.nytimes.com/svc/search/v2/articlesearch.json?q={query}&page={i}&api-key={key}'
response = requests.get(url)
parsed = response.json()
articles = parsed['response']['docs']
results.append(articles)
sleep(6) # sleep at least 6 seconds not to overload the servers
In the end, we have a list of results, 5 lists to be exact (each list represents one page of articles). Each result (or page) contains information about 10 articles that matched our search.
type(results)
list
len(results)
5
type(results[0])
list
len(results[0])
10
pulling articles out of our results#
Let’s try to look at just the first article on the first page. To do that, we need to know what kind of data we are dealing with. Use type() and list and dict indexing methods to figure out how to get just the abstract from the first article.
type(results)
list
type(results[0])
list
# we have a dict within a list within a list
type(results[0][0])
dict
results[0][0].keys()
dict_keys(['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'subsection_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri'])
results[0][0]['abstract']
'There has been a dramatic drop in the number of people gathering at the U.S. border and trying to cross. Can it help Mexico stave off President Trump’s threatened tariffs?'
We can simplify our life by creating a variable
article = results[0][0]
individual challenge: exploring keys#
Explore some of the other keys, in this article object. Some of them contain more nesting dicts. See if you can access the data inside of them.
article.keys()
dict_keys(['abstract', 'web_url', 'snippet', 'lead_paragraph', 'print_section', 'print_page', 'source', 'multimedia', 'headline', 'keywords', 'pub_date', 'document_type', 'news_desk', 'section_name', 'subsection_name', 'byline', 'type_of_material', '_id', 'word_count', 'uri'])
article['byline']
{'original': 'By Annie Correal and Alejandro Cegarra',
'person': [{'firstname': 'Annie',
'middlename': None,
'lastname': 'Correal',
'qualifier': None,
'title': None,
'role': 'reported',
'organization': '',
'rank': 1},
{'firstname': 'Alejandro',
'middlename': None,
'lastname': 'Cegarra',
'qualifier': None,
'title': None,
'role': 'reported',
'organization': '',
'rank': 2}],
'organization': None}
type(article['byline'])
dict
article['byline'].keys()
dict_keys(['original', 'person', 'organization'])
article['byline']['person'][0]['firstname']
'Annie'
article['byline']['person'][0]['lastname']
'Correal'
saving our data to lists#
Now let’s say we want just a list of the abstracts. We could write a loop (actually, a loop within a loop!) to pull out the abstract information for each article on each page.
# we need to do nested loops
for result in results: # loops through list of pages
for article in result: # loops through list of articles
print(article['abstract']) # grabs abstract of article
There has been a dramatic drop in the number of people gathering at the U.S. border and trying to cross. Can it help Mexico stave off President Trump’s threatened tariffs?
The new case, which for now is asking for a court to block the transfer of 10 men to the offshore base, is the first to directly challenge the policy.
The missing people were on two boats that capsized off Yemen, which is on a major route for migrants trying to reach Gulf countries for work.
President Trump said the base would house as many as 30,000 migrants awaiting deportation. But construction of a tent city was halted weeks ago.
More than 100 asylum seekers from around the world sent to Panama from the United States have no sense how long they will be held or where they might eventually be sent.
Mayor Eric Adams announced on Monday that the hotel will stop housing migrants by June, calling the closure a milestone in New York City’s response to the crisis.
A transfer operation on Thursday repatriated 177 Venezuelans via a handoff in Honduras, while one migrant was brought back to U.S. soil.
At a House hearing, the mayors of Boston, Chicago, Denver and New York City rejected accusations that they were harboring criminals by not aiding deportation efforts.
The president signed an order that would deny loan forgiveness to workers for groups engaged in “substantial illegal activities,” which it indicated included things like diversity initiatives.
Aiming to appease President Trump, Mexico is hitting China with tariffs, handing cartel leaders over to the United States and using C.I.A. intelligence to hunt down others.
The Canadian government has reinforced border operations to stop migrants going to the United States, a major Trump complaint. But early data shows people are, instead, starting to flee the United States for Canada.
Lawsuits are challenging President Trump’s abrupt decision to send men awaiting deportation to the American military base in Cuba.
In Corona, a once-vibrant New York City enclave home to generations of Latino immigrants, the threat of mass deportations has made many residents afraid to go outside.
The Trump administration has said little about the Venezuelan men who were transferred from Texas to the U.S. military base in Cuba.
President Trump’s executive order also directed federal departments and agencies to ensure that federal funds do not encourage people to come to the United States illegally.
The group of unauthorized migrants, which includes children, were bused to the camp late Tuesday night. “It looks like a zoo, there are fenced cages,” said one of the detainees.
Costa Rica is the second Central American nation to accept migrants from distant countries as the Trump administration ramps up deportation flights.
Most undocumented immigrants are ineligible for federal benefits already. But here’s what the directive could mean in practice.
A new policy requires the fingerprinting of all adults who will be living with a migrant child in the United States. Advocates for immigrants say it could fuel overcrowding in shelters.
Plus, meat raffles: like bingo, but with beef.
The decision to sue over the $80 million in seized funds comes as the New York City mayor has been accused of supporting the White House’s immigration agenda in exchange for legal leniency.
Days after the United States sent 300 migrants from Asia and the Middle East to Panama, a Panamanian official said that more than half had agreed to be deported to their countries of origin.
The move would be a drastic escalation by the White House to militarize immigration enforcement.
The administration has asked Panama to take in hundreds of people who can’t easily be sent back to their countries. Many say they are in danger.
As India’s prime minister, Narendra Modi, meets with President Trump, issues around immigration are likely to be a source of tension, including a U.S. military flight that returned shackled migrants to India.
The move could herald a new front in the Trump administration’s plans for mass deportations, one that allows for more rapid removal of migrants whose home countries are reluctant to accept them.
The Times has obtained a list of 53 Venezuelan men the Trump administration has put in a wartime prison built to hold Al Qaeda suspects.
Luis Alberto Castillo arrived in the United States so that he could “give everything to his son,” said his sister. Then, while scrolling on TikTok, she found out he was headed to Guantánamo.
The flights, which left Fort Bliss in Texas, represent a significant victory for President Trump, who has vowed to carry out sweeping deportations.
The city is dramatically rewriting the story of skilled migration.
The legislation would increase penalties for citizens and allow anyone who is not a citizen to be deported for evading arrest or detention while driving a vehicle within 100 miles of the border.
Talking to people around Chicago, I heard the word “resentment” over and over.
In a lawsuit, the groups asked a judge to enable lawyers to communicate with the men in case they would like legal representation.
Fearing roundups, many immigrants are staying home. Construction, agriculture, senior care and hospitality employers say labor shortages will worsen.
Mayor Mike Johnston faced hours of hostile questions from House members, some from his own party.
The president should reread the Declaration of Independence.
My decision to leave the U.S. seemed crazy. I believed then that America was exceptional, the only nation capable of caring for its people.
President Trump and his divisive policies are helping to define the primary races for New Jersey governor.
The judges, who are part of the Justice Department, make decisions about asylum claims and have the power to order someone removed from the country.
Plus, Dolly Parton’s love story.
Republicans accused the mayors of New York, Chicago, Denver and Boston of providing sanctuary to criminals, while Democrats pointed to falling crime rates and defended helping the needy.
Democratic lawmakers grilled Mayor Eric Adams of New York City over the circumstances behind the Justice Department’s move to dismiss his criminal case.
President Trump’s promise to launch the largest deportation operation in U.S. history is colliding with the practical difficulties of detaining people and transporting them across the globe.
What Times Opinion columnists and writers thought of the president’s speech to Congress.
At 9 p.m. Eastern, President Trump will speak to a joint session of Congress for the first time in his second term. The New York Times will carry the address live.
In a letter to Attorney General Pam Bondi, the lawmakers, Jamie Raskin of Maryland and Jasmine Crockett of Texas, accused the Justice Department of a coverup.
A group of lawyers accused Panama of violating the rights of people deported from the United States under the Trump administration’s policy of sending migrants to cooperative Latin American nations.
President Trump’s plan to apply 25 percent tariffs on Canadian exports is set to start on Tuesday. They would deal a brutal blow to Canada’s economy.
The Times has obtained a list of the names of the men, whom the U.S. government has described as Venezuelan citizens under final deportation orders.
The troop mobilization indicates that President Trump is breaking with recent presidents’ practice of limiting deployments along the U.S.-Mexico border mostly to small numbers of active-duty soldiers and reservists.
group challenge: lists more lists!#
Go through the article data, and grab the following information:
abstract
publication date
one other detail (check the list of
keys()) of your choosing
Save all of that information to lists.
abstracts = []
dates = []
keywords = []
for result in results:
for item in result:
abstracts.append(item['abstract'])
dates.append(item['pub_date'])
keywords.append(item['keywords'])
len(abstracts)
50
abstracts[:10]
['There has been a dramatic drop in the number of people gathering at the U.S. border and trying to cross. Can it help Mexico stave off President Trump’s threatened tariffs?',
'The new case, which for now is asking for a court to block the transfer of 10 men to the offshore base, is the first to directly challenge the policy.',
'The missing people were on two boats that capsized off Yemen, which is on a major route for migrants trying to reach Gulf countries for work.',
'President Trump said the base would house as many as 30,000 migrants awaiting deportation. But construction of a tent city was halted weeks ago.',
'More than 100 asylum seekers from around the world sent to Panama from the United States have no sense how long they will be held or where they might eventually be sent.',
'Mayor Eric Adams announced on Monday that the hotel will stop housing migrants by June, calling the closure a milestone in New York City’s response to the crisis.',
'A transfer operation on Thursday repatriated 177 Venezuelans via a handoff in Honduras, while one migrant was brought back to U.S. soil.',
'At a House hearing, the mayors of Boston, Chicago, Denver and New York City rejected accusations that they were harboring criminals by not aiding deportation efforts.',
'The president signed an order that would deny loan forgiveness to workers for groups engaged in “substantial illegal activities,” which it indicated included things like diversity initiatives.',
'Aiming to appease President Trump, Mexico is hitting China with tariffs, handing cartel leaders over to the United States and using C.I.A. intelligence to hunt down others.']
Bonus: going deep into keywords#
for result in results:
for item in result:
print(item['keywords'][0]['value'])
United States International Relations
Guantanamo Bay Naval Base (Cuba)
Drownings
United States Defense and Military Forces
Immigration Detention
Immigration Shelters
Trump, Donald J
United States Politics and Government
United States Politics and Government
Illegal Immigration
Immigration and Emigration
Illegal Immigration
Corona (Queens, NY)
Guantanamo Bay Naval Base (Cuba)
United States Politics and Government
Illegal Immigration
Costa Rica
United States Politics and Government
United States Politics and Government
Aviation Accidents, Safety and Disasters
New York City
Panama
United States Defense and Military Forces
Illegal Immigration
Deportation
Deportation
United States Defense and Military Forces
Illegal Immigration
Illegal Immigration
Foreign Workers
United States Politics and Government
Trump, Donald J
Illegal Immigration
Foreign Workers
Johnston, Michael (1974- )
Trump, Donald J
Illegal Immigration
United States Politics and Government
United States Politics and Government
United States Politics and Government
Mayors
Adams, Eric L
Illegal Immigration
United States Politics and Government
State of the Union Message (US)
United States Politics and Government
Deportation
Protectionism (Trade)
United States Defense and Military Forces
United States Defense and Military Forces
abstracts = []
dates = []
keywords = []
for result in results:
for item in result:
abstracts.append(item['abstract'])
dates.append(item['pub_date'])
keywords.append(item['keywords'][0]['value'])
individual challenge: to dataframe!#
make the lists into a dataframe
import pandas as pd
df = pd.DataFrame({
'date': dates,
'abstract': abstracts,
'keywords': keywords
})
df.head()
| date | abstract | keywords | |
|---|---|---|---|
| 0 | 2025-03-03T14:00:34+0000 | There has been a dramatic drop in the number o... | United States International Relations |
| 1 | 2025-03-01T19:34:05+0000 | The new case, which for now is asking for a co... | Guantanamo Bay Naval Base (Cuba) |
| 2 | 2025-03-07T12:33:25+0000 | The missing people were on two boats that caps... | Drownings |
| 3 | 2025-03-10T20:20:40+0000 | President Trump said the base would house as m... | United States Defense and Military Forces |
| 4 | 2025-02-28T22:01:42+0000 | More than 100 asylum seekers from around the w... | Immigration Detention |
df.to_csv('nyt_data.csv')
text analysis with spaCy#
import spacy
nlp = spacy.load('en_core_web_sm')
docs = list(nlp.pipe(abstracts))
for doc in docs:
print(doc)
The decision comes as political pressure mounts to cut down on programs that allow migrants to stay in the United States temporarily, even without a visa or green card.
The country’s defense ministry said the military officers who opened fire might have mistaken the migrants for cartel members.
Italy, an aging country, badly needs migrant labor and immigration, but the government has admitted that the pathways for legal entry are rife with abuse.
At least nine people are known to have died in a sinking off the Canary Islands and 48 more are missing, the latest disaster on the perilous Atlantic crossing from West Africa.
It’s been a year since Mayor Eric Adams made his ominous prediction. City officials should treat the arrival of migrants as an opportunity rather than a catastrophe, advocates say.
Reports and interviews shed new light on the holding center, where migrants’ calls with lawyers are monitored and some say they’ve been forced to wear blackout goggles.
A group of New York’s most powerful philanthropies will spend millions to help make migrants more self-sufficient.
Former President Donald J. Trump has long tried to stoke fear about immigration. Now, he’s doubling down as the presidential race heats up.
In his next documentary, Michael P. Nash takes on A.I. and how it might be used to address environmental issues.
The former president and his allies, who often decry undocumented immigrants, are targeting programs that allow millions of people to enter the country lawfully.
A federal appeals court will consider the future of DACA, which has allowed hundreds of thousands of undocumented young people to live and work in the United States.
Donald J. Trump answered preselected questions from a pro-military crowd who echoed his false claims and approved of his vow to conduct massive deportations of undocumented immigrants.
Several countries have chartered flights, while others issued warnings and offered assistance.
For the former president, removing millions of undocumented immigrants is the solution to most of the nation’s most pressing challenges.
An ad from two Democratic groups skates over Kamala Harris’s tough recent stance on border security and instead focuses on her more immigration-friendly plans.
Gov. Tim Walz of Minnesota has acted as attack dog for the Harris campaign, and Senator JD Vance of Ohio has stepped in to explain away former President Donald J. Trump’s incendiary remarks.
Germany’s unilateral decision to close its borders has angered its neighbors in Europe.
Eric Adams gives new meaning to the concept of the frequent-flier program.
Vice President Kamala Harris vowed to combat the flow of fentanyl into the country when she visited the border on Friday. Former President Donald J. Trump ratcheted up his false claims in response.
Former President Donald J. Trump assailed Vice President Kamala Harris in harshly personal terms in what he conceded was “a dark speech.”
The Freedom Party has made itself the country’s most popular party, with calls to bar asylum seekers. It is poised to come out on top in parliamentary elections for the first time.
Donald Trump used a pair of events to try to blame Kamala Harris for inflation and the migrant crisis, tapping into themes that helped him win Michigan in 2016.
A local official said there was “absolutely no evidence” that Haitian migrants have stolen and eaten pets, an outlandish claim amplified by the Trump campaign on Monday.
The change will make what had been a short-term fix, which appears to have led to a drop in border crossings, a central feature of the asylum system.
Vowing to carry on President Biden’s crackdown on asylum and to impose order on the southern border, the vice president demonstrated how much the politics of immigration have changed.
Visiting the border on Friday in Arizona, the vice president will try to neutralize a key vulnerability, and will demonstrate her party’s harder line on immigration.
The Republican vice-presidential nominee acknowledged that an outlandish claim he had amplified about Haitian immigrants stealing and eating pets could “turn out to be false.”
In his first interview with Western media since becoming president of Senegal, Bassirou Diomaye Faye said the United Nations has to change to reflect changing world demographics.
During a trip to Arizona, Vice President Kamala Harris is expected to make her first visit to the southern border since becoming the Democratic nominee.
Mr. Musk described Prime Minister Giorgia Meloni as “authentic, honest and thoughtful.” She used her Atlantic Council spotlight to defend Western values.
From calling for mass deportations to spreading false claims about migrants’ eating pets, former President Donald J. Trump and his running mate, Senator JD Vance of Ohio, are taking a hard line.
Tren de Aragua’s emergence in New York City and elsewhere has led law enforcement officials to strategize how to stem the gang’s growth.
A gallery in a shelter near the Brooklyn Navy Yard is showcasing the work of two migrants. One of them did some of his paintings under the Brooklyn-Queens Expressway.
The immigrants are mainly in the United States under temporary protected status, which the executive branch can grant to people whose home countries are in crisis.
We cover the presidential candidates’ immigration policies.
Margarita Solito and her family fled violence and poverty in El Salvador, hoping to build a better life in San Francisco. The city often wasn’t what they thought it would be.
Italian rescuers said that seven Syrian migrants survived a shipwreck about 10 miles from the island of Lampedusa, in the Central Mediterranean. The survivors told officials that 21 fellow travelers, including three minors, had fallen off during the crossing because of rough seas.
The Springfield fable is part of a crusade to throw immigrants out of the country.
Seven survivors were rescued from the boat that sank after leaving Libya. A crackdown by Italy has reduced migration, but not the high death rates that come with crossing the Mediterranean.
Representative Gabe Vasquez, a Democrat in a border district, is under attack by his Republican opponent on immigration policy. Can a pro-abortion access message help him prevail?
The former president meandered over 80 minutes from complaints about Tuesday’s debate to elevating baseless claims that have circulated on right-wing social media.
Secretary of Homeland Security Alejandro Mayorkas explains what his agency has done to address the challenges at the U.S.-Mexico border.
Speaking at an event in Tucson, Ariz., former President Donald J. Trump called for eliminating taxes on overtime pay as he declared Tuesday’s debate a “monumental victory” for him.
Binyamin Appelbaum on why Harris’s ill-defined plan is still better than Trump’s.
Apparently, this election is about … pets.
Over 60 people were crowded into a small boat that “rapidly and brutally” broke apart off the coast of France, authorities said.
Our columnists and contributors rate Trump’s and Harris’s debate performances.
Former President Donald J. Trump offered a dire portrait of America, often relying on false and debunked claims as he described “a failing nation.”
The government said it would expand patrols as part of a ‘hard line’ on illegal migration following two deadly stabbings by rejected asylum seekers and a surge in support for the far right.
As the candidates meet for their only scheduled debate on Tuesday night, here is what to be on the lookout for.
tokens#
for token in docs[0]:
print(token)
The
decision
comes
as
political
pressure
mounts
to
cut
down
on
programs
that
allow
migrants
to
stay
in
the
United
States
temporarily
,
even
without
a
visa
or
green
card
.
type(token)
spacy.tokens.token.Token
# see all the methods for Token objects!
dir(token)
['_',
'__bytes__',
'__class__',
'__delattr__',
'__dir__',
'__doc__',
'__eq__',
'__format__',
'__ge__',
'__getattribute__',
'__getstate__',
'__gt__',
'__hash__',
'__init__',
'__init_subclass__',
'__le__',
'__len__',
'__lt__',
'__ne__',
'__new__',
'__pyx_vtable__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__setattr__',
'__sizeof__',
'__str__',
'__subclasshook__',
'__unicode__',
'ancestors',
'check_flag',
'children',
'cluster',
'conjuncts',
'dep',
'dep_',
'doc',
'ent_id',
'ent_id_',
'ent_iob',
'ent_iob_',
'ent_kb_id',
'ent_kb_id_',
'ent_type',
'ent_type_',
'get_extension',
'has_dep',
'has_extension',
'has_head',
'has_morph',
'has_vector',
'head',
'i',
'idx',
'iob_strings',
'is_alpha',
'is_ancestor',
'is_ascii',
'is_bracket',
'is_currency',
'is_digit',
'is_left_punct',
'is_lower',
'is_oov',
'is_punct',
'is_quote',
'is_right_punct',
'is_sent_end',
'is_sent_start',
'is_space',
'is_stop',
'is_title',
'is_upper',
'lang',
'lang_',
'left_edge',
'lefts',
'lemma',
'lemma_',
'lex',
'lex_id',
'like_email',
'like_num',
'like_url',
'lower',
'lower_',
'morph',
'n_lefts',
'n_rights',
'nbor',
'norm',
'norm_',
'orth',
'orth_',
'pos',
'pos_',
'prefix',
'prefix_',
'prob',
'rank',
'remove_extension',
'right_edge',
'rights',
'sent',
'sent_start',
'sentiment',
'set_extension',
'set_morph',
'shape',
'shape_',
'similarity',
'subtree',
'suffix',
'suffix_',
'tag',
'tag_',
'tensor',
'text',
'text_with_ws',
'vector',
'vector_norm',
'vocab',
'whitespace_']
for token in docs[0]:
print(token, token.pos_, token.dep_)
The DET det
decision NOUN nsubj
comes VERB ROOT
as ADP prep
political ADJ amod
pressure NOUN compound
mounts NOUN pobj
to PART aux
cut VERB relcl
down ADP prt
on ADP prep
programs NOUN pobj
that PRON nsubj
allow VERB relcl
migrants NOUN nsubj
to PART aux
stay VERB ccomp
in ADP prep
the DET det
United PROPN compound
States PROPN pobj
temporarily ADV advmod
, PUNCT punct
even ADV advmod
without ADP prep
a DET det
visa NOUN pobj
or CCONJ cc
green ADJ amod
card NOUN conj
. PUNCT punct
NER#
# see docs (hah!) on Doc object on this page: https://spacy.io/api/doc
type(doc)
spacy.tokens.doc.Doc
dir(doc)
['_',
'__bytes__',
'__class__',
'__delattr__',
'__dir__',
'__doc__',
'__eq__',
'__format__',
'__ge__',
'__getattribute__',
'__getitem__',
'__getstate__',
'__gt__',
'__hash__',
'__init__',
'__init_subclass__',
'__iter__',
'__le__',
'__len__',
'__lt__',
'__ne__',
'__new__',
'__pyx_vtable__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__setattr__',
'__setstate__',
'__sizeof__',
'__str__',
'__subclasshook__',
'__unicode__',
'_bulk_merge',
'_context',
'_get_array_attrs',
'_realloc',
'_vector',
'_vector_norm',
'cats',
'char_span',
'copy',
'count_by',
'doc',
'ents',
'extend_tensor',
'from_array',
'from_bytes',
'from_dict',
'from_disk',
'from_docs',
'from_json',
'get_extension',
'get_lca_matrix',
'has_annotation',
'has_extension',
'has_unknown_spaces',
'has_vector',
'is_nered',
'is_parsed',
'is_sentenced',
'is_tagged',
'lang',
'lang_',
'mem',
'noun_chunks',
'noun_chunks_iterator',
'remove_extension',
'retokenize',
'sentiment',
'sents',
'set_ents',
'set_extension',
'similarity',
'spans',
'tensor',
'text',
'text_with_ws',
'to_array',
'to_bytes',
'to_dict',
'to_disk',
'to_json',
'to_utf8_array',
'user_data',
'user_hooks',
'user_span_hooks',
'user_token_hooks',
'vector',
'vector_norm',
'vocab']
for doc in docs:
for ent in doc.ents:
print(ent.text, ent.label_)
the United States GPE
Italy GPE
At least nine CARDINAL
the Canary Islands LOC
48 CARDINAL
Atlantic LOC
West Africa GPE
Eric Adams PERSON
New York GPE
millions CARDINAL
Donald J. Trump PERSON
Michael P. Nash PERSON
A.I. ORG
millions CARDINAL
hundreds of thousands CARDINAL
the United States GPE
Donald J. Trump PERSON
millions CARDINAL
two CARDINAL
Democratic NORP
Kamala Harris’s PERSON
Tim Walz PERSON
Minnesota GPE
Harris PERSON
JD Vance PERSON
Ohio GPE
Donald J. Trump’s PERSON
Germany GPE
Europe LOC
Eric Adams PERSON
Kamala Harris PERSON
Friday DATE
Donald J. Trump PERSON
Donald J. Trump PERSON
Kamala Harris PERSON
The Freedom Party ORG
first ORDINAL
Donald Trump PERSON
Kamala Harris PERSON
Michigan GPE
2016 DATE
Haitian NORP
Trump ORG
Monday DATE
Biden PERSON
Friday DATE
Arizona GPE
Republican NORP
Haitian NORP
first ORDINAL
Western NORP
Senegal GPE
Bassirou Diomaye Faye PERSON
the United Nations ORG
Arizona GPE
Kamala Harris PERSON
first ORDINAL
Democratic NORP
Musk PERSON
Giorgia Meloni PERSON
Atlantic Council ORG
Western NORP
Donald J. Trump PERSON
JD Vance PERSON
Ohio GPE
New York City GPE
the Brooklyn Navy Yard ORG
two CARDINAL
One CARDINAL
the Brooklyn-Queens Expressway ORG
the United States GPE
Margarita Solito PERSON
El Salvador GPE
San Francisco GPE
Italian NORP
seven CARDINAL
Syrian NORP
about 10 miles QUANTITY
Lampedusa PERSON
the Central Mediterranean LOC
21 CARDINAL
three CARDINAL
Springfield GPE
Seven CARDINAL
Libya GPE
Italy GPE
Mediterranean LOC
Gabe Vasquez PERSON
Democrat NORP
Republican NORP
80 minutes TIME
about Tuesday DATE
Homeland Security Alejandro Mayorkas ORG
U.S.-Mexico ORG
Tucson GPE
Ariz. GPE
Donald J. Trump PERSON
Tuesday DATE
Binyamin Appelbaum PERSON
Harris’s PERSON
Trump’s ORG
60 CARDINAL
France GPE
Trump’s ORG
Harris PERSON
Donald J. Trump PERSON
America GPE
two CARDINAL
Tuesday DATE
night TIME
word frequencies#
from collections import Counter
words = []
for doc in docs:
for token in doc:
if not token.is_stop:
if not token.is_punct:
words.append(token.text)
word_freq = Counter(words)
common_words = word_freq.most_common(20)
print(common_words)
[('Trump', 12), ('President', 11), ('migrants', 9), ('Donald', 9), ('J.', 8), ('Harris', 8), ('border', 8), ('immigration', 7), ('country', 6), ('said', 6), ('president', 6), ('immigrants', 6), ('people', 5), ('false', 5), ('claims', 5), ('Kamala', 5), ('United', 4), ('undocumented', 4), ('asylum', 4), ('pets', 4)]
dependency#
from spacy import displacy
displacy.render(docs[0], style="dep", jupyter=True)
BONUS: searching text by grammatical dependancy#
See this tutorial for more info: https://applied-language-technology.mooc.fi/html/notebooks/part_iii/03_pattern_matching.html
# first, join the docs into one string to process with matcher
# then run nlp() again
abstracts_string = ' '.join(abstracts)
doc = nlp(abstracts_string)
doc[:50]
The decision comes as political pressure mounts to cut down on programs that allow migrants to stay in the United States temporarily, even without a visa or green card. The country’s defense ministry said the military officers who opened fire might have mistaken the migrants for cartel
from spacy.matcher import Matcher
# Create a Matcher and provide model vocabulary; assign result under the variable 'matcher'
matcher = Matcher(nlp.vocab)
# Call the variable to examine the object
matcher
<spacy.matcher.matcher.Matcher at 0x320449cf0>
# Define a list with nested dictionaries that contains the pattern to be matched
pronoun_verb = [{'POS': 'PRON'}, {'POS': 'VERB'}]
# Add the pattern to the matcher under the name 'pronoun+verb'
matcher.add("pronoun+verb", patterns=[pronoun_verb])
# Apply the Matcher to the Doc object under 'doc'; provide the argument
# 'as_spans' and set its value to True to get Spans as output\
matches = matcher(doc)
# Call the variable to examine the output
# result
matches
[(12298179334642351811, 12, 14),
(12298179334642351811, 40, 42),
(12298179334642351811, 168, 170),
(12298179334642351811, 265, 267),
(12298179334642351811, 318, 320),
(12298179334642351811, 482, 484),
(12298179334642351811, 518, 520),
(12298179334642351811, 586, 588),
(12298179334642351811, 588, 590),
(12298179334642351811, 598, 600),
(12298179334642351811, 638, 640),
(12298179334642351811, 822, 824),
(12298179334642351811, 916, 918),
(12298179334642351811, 957, 959),
(12298179334642351811, 995, 997),
(12298179334642351811, 1072, 1074),
(12298179334642351811, 1092, 1094),
(12298179334642351811, 1129, 1131),
(12298179334642351811, 1202, 1204),
(12298179334642351811, 1300, 1302)]
for match_id, start, end in matches:
string_id = doc.vocab.strings[match_id] # Look up string ID
span = doc[start:end]
print(string_id, span.text)
pronoun+verb that allow
pronoun+verb who opened
pronoun+verb some say
pronoun+verb that allow
pronoun+verb who echoed
pronoun+verb she visited
pronoun+verb he conceded
pronoun+verb that helped
pronoun+verb him win
pronoun+verb there was
pronoun+verb which appears
pronoun+verb She used
pronoun+verb them did
pronoun+verb We cover
pronoun+verb they thought
pronoun+verb that sank
pronoun+verb that come
pronoun+verb him prevail
pronoun+verb he declared
pronoun+verb he described