pandas_to_vertica

In [ ]:
pandas_to_vertica(df, 
                  name: str = "", 
                  schema: str = "", 
                  dtype: dict = {}, 
                  parse_nrows: int = 10000, 
                  temp_path: str = "", 
                  insert: bool = False,)

Ingests a pandas DataFrame into the Vertica database by creating a CSV file and then using flex tables.

Parameters

Name Type Optional Description
df
pandas.DataFrame
The pandas.DataFrame to ingest.
name
str
Name of the new relation, or an existing relation, in which to insert the data. If unspecified, the temporary schema will be used. To change the default relation, use the set_option function.
schema
str
Schema of the new relation. If unspecified, a temporary table is created.
dtype
dict
Dictionary of input types. Providing a dictionary can increase ingestion speed and precision. If specified, VerticaPy uses the designated input types instead of parsing the intermediate CSV and inferring the input types.
parse_nrows
int
If this parameter is greater than 0, VerticaPy creates and ingests a temporary file containing 'parse_nrows' lines to determine the input data types. VerticaPy then ingests the remainder of the intermediate CSV file containing the rest of the data. This method of data type identification is less accurate, but much faster for large datasets.
temp_path
str
The path to which the intermediate CSV file is written. This is useful in cases where the user does not have write permissions on the current directory.
insert
bool
If set to True, the data is ingested into the input relation. The column names of your table and the pandas.DataFrame must match.

Returns

vDataFrame : vDataFrame of the new relation.

Example

In [19]:
from verticapy.datasets import load_titanic
titanic = load_titanic()
df = titanic.to_pandas()
df
Out[19]:
pclass survived name sex age sibsp parch ticket fare cabin embarked boat body home.dest
0 1 0 Allison, Miss. Helen Loraine female 2.000 1 2 113781 151.55000 C22 C26 S None NaN Montreal, PQ / Chesterville, ON
1 1 0 Allison, Mr. Hudson Joshua Creighton male 30.000 1 2 113781 151.55000 C22 C26 S None 135.0 Montreal, PQ / Chesterville, ON
2 1 0 Allison, Mrs. Hudson J C (Bessie Waldo Daniels) female 25.000 1 2 113781 151.55000 C22 C26 S None NaN Montreal, PQ / Chesterville, ON
3 1 0 Andrews, Mr. Thomas Jr male 39.000 0 0 112050 0.00000 A36 S None NaN Belfast, NI
4 1 0 Artagaveytia, Mr. Ramon male 71.000 0 0 PC 17609 49.50420 None C None 22.0 Montevideo, Uruguay
... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
1229 3 1 Wilkes, Mrs. James (Ellen Needs) female 47.000 1 0 363272 7.00000 None S None NaN None
1230 3 1 Yasbeck, Mrs. Antoni (Selini Alexander) female 15.000 1 0 2659 14.45420 None C None NaN None
1231 3 1 de Messemaeker, Mr. Guillaume Joseph male 36.500 1 0 345572 17.40000 None S 15 NaN Tampico, MT
1232 3 1 de Messemaeker, Mrs. Guillaume Joseph (Emma) female 36.000 1 0 345572 17.40000 None S 13 NaN Tampico, MT
1233 3 1 de Mulder, Mr. Theodore male 30.000 0 0 345774 9.50000 None S 11 NaN Belgium Detroit, MI

1234 rows × 14 columns

In [3]:
from verticapy.utilities import *
pandas_to_vertica(df = df, name = "titanic_pandas", schema = "public")
123
fare
Numeric(10,5)
123
survived
Int
Abc
sex
Varchar(20)
Abc
boat
Varchar(100)
123
pclass
Int
123
age
Numeric(6,3)
Abc
ticket
Varchar(36)
Abc
Varchar(164)
Abc
embarked
Varchar(20)
Abc
cabin
Varchar(30)
123
body
Int
123
parch
Int
Abc
home.dest
Varchar(100)
123
sibsp
Int
1151.550000female[null]12.000113781SC22 C26[null]2Montreal, PQ / Chesterville, ON1
2151.550000male[null]130.000113781SC22 C261352Montreal, PQ / Chesterville, ON1
3151.550000female[null]125.000113781SC22 C26[null]2Montreal, PQ / Chesterville, ON1
40.000000male[null]139.000112050SA36[null]0Belfast, NI0
549.504200male[null]171.000PC 17609C[null]220Montevideo, Uruguay0
6227.525000male[null]147.000PC 17757CC62 C641240New York, NY1
725.925000male[null]1[null]PC 17318S[null][null]0New York, NY0
8247.520800male[null]124.000PC 17558CB58 B60[null]1Montreal, PQ0
975.241700maleA136.00013050CC6[null]0Winnipeg, MN0
1026.000000male[null]125.00013905C[null]1480San Francisco, CA0
1135.500000male[null]145.000113784ST[null]0Trenton, NJ0
1226.550000male[null]142.000110489SD22[null]0London / Winnipeg, MB0
1330.500000male[null]141.000113054SA21[null]0Pomeroy, WA0
1450.495800male[null]148.000PC 17591CB102080Omaha, NE0
1539.600000male[null]1[null]112379C[null][null]0Philadelphia, PA0
1626.550000male[null]145.000113050SB38[null]0Washington, DC0
1731.000000male[null]1[null]113798S[null][null]0[null]0
185.000000male[null]133.000695SB51 B53 B55[null]0New York, NY0
1947.100000male[null]128.000113059S[null][null]0Montevideo, Uruguay0
2047.100000male[null]117.000113059S[null][null]0Montevideo, Uruguay0
2126.000000male[null]149.00019924S[null][null]0Ascot, Berkshire / Rochester, NY0
2278.850000male[null]136.00019877SC461720Little Onn Hall, Staffs1
2361.175000male[null]146.000W.E.P. 5734SE31[null]0Amenia, ND1
240.000000male[null]1[null]112051S[null][null]0Liverpool, England / Belfast0
25136.779200male[null]127.00013508CC89[null]0Los Angeles, CA1
2652.000000male[null]1[null]110465SA14[null]0Stoughton, MA0
2725.587500male[null]147.0005727SE58[null]0Victoria, BC0
2883.158300male[null]137.000PC 17756CE52[null]1Lakewood, NJ1
2926.550000male[null]1[null]113791S[null][null]0Roachdale, IN0
3071.000000male[null]170.000WE/P 5735SB222691Milwaukee, WI1
3171.283300male[null]139.000PC 17599CC85[null]0New York, NY1
3252.000000male[null]131.000F.C. 12750SB71[null]0Montreal, PQ1
33106.425000male[null]150.000PC 17761CC86620Deephaven, MN / Cedar Rapids, IA1
3429.700000male[null]139.000PC 17580CA181330Philadelphia, PA0
3531.679200female[null]136.000PC 17531CA29[null]0New York, NY0
36221.779200male[null]1[null]PC 17483SC95[null]0[null]0
3727.750000male[null]130.000113051CC111[null]0New York, NY0
38263.000000male[null]119.00019950SC23 C25 C27[null]2Winnipeg, MB3
39263.000000male[null]164.00019950SC23 C25 C27[null]4Winnipeg, MB1
4026.550000male[null]1[null]113778SD34[null]0Westcliff-on-Sea, Essex0
410.000000male[null]1[null]112058SB102[null]0[null]0
4253.100000male[null]137.000113803SC123[null]0Scituate, MA1
4338.500000male[null]147.000111320SE632750St Anne's-on-Sea, Lancashire0
4479.200000male[null]124.000PC 17593CB86[null]0[null]0
4534.654200male[null]171.000PC 17754CA5[null]0New York, NY0
46153.462500male[null]138.000PC 17582SC911471Winnipeg, MB0
4779.200000male[null]146.000PC 17593CB82 B84[null]0New York, NY0
4842.400000male[null]1[null]113796S[null][null]0[null]0
4983.475000male[null]145.00036973SC83[null]0New York, NY1
500.000000male[null]140.000112059SB941100[null]0
5193.500000male[null]155.00012749SB693071Montreal, PQ1
5242.500000male[null]142.000113038SB11[null]0London / Middlesex0
5351.862500male[null]1[null]17463SE46[null]0Brighton, MA0
5450.000000male[null]155.000680SC39[null]0London / Birmingham0
5552.000000male[null]142.000113789S[null]380New York, NY1
5630.695800male141[null]PC 17600C[null][null]0New York, NY0
5728.712500female[null]150.000PC 17595CC49[null]0Paris, France New York, NY0
5826.000000male[null]146.000694S[null]800Bennington, VT0
5926.000000male[null]150.000113044SE60[null]0London0
60211.500000male[null]132.500113503CC132450[null]0
6129.700000male[null]158.00011771CB372580Buffalo, NY0
6251.862500male[null]141.00017464SD21[null]0Southington / Noank, CT1
6326.550000male[null]1[null]113028SC124[null]0Portland, OR0
6427.720800male[null]1[null]PC 17612C[null][null]0Chicago, IL0
6530.000000male[null]129.000113501SD61260Springfield, MA0
6645.500000male[null]130.000113801S[null][null]0London / New York, NY0
6726.000000male[null]130.000110469SC106[null]0Brockton, MA0
6853.100000male[null]119.000113773SD30[null]0New York, NY1
6975.241700male[null]146.00013050CC62920Vancouver, BC0
7051.862500male[null]154.00017463SE461750Dorchester, MA0
7182.170800male[null]128.000PC 17604C[null][null]0New York, NY1
7226.550000male[null]165.00013509SE382490East Bridgewater, MA0
7390.000000male[null]144.00019928QC782300Fond du Lac, WI2
7430.500000male[null]155.000113787SC30[null]0Montreal, PQ0
7542.400000male[null]147.000113796S[null][null]0Washington, DC0
7629.700000male[null]137.000PC 17596CC118[null]1Brooklyn, NY0
77113.275000male[null]158.00035273CD481222Lexington, MA0
7826.000000male[null]164.000693S[null]2630Isle of Wight, England0
7961.979200male[null]165.000113509CB302341Providence, RI0
8027.720800male[null]128.500PC 17562CD431890?Havana, Cuba0
810.000000male[null]1[null]112052S[null][null]0Belfast0
8228.500000male[null]145.500113043SC1241660Surbiton Hill, Surrey0
8393.500000male[null]123.00012749SB24[null]0Montreal, PQ0
8466.600000male[null]129.000113776SC2[null]0Isleworth, England1
85108.900000male[null]118.000PC 17758CC65[null]0Madrid, Spain1
8652.000000male[null]147.000110465SC1102070Worcester, MA0
870.000000male[null]138.00019972S[null][null]0Rotterdam, Netherlands0
88135.633300male[null]122.000PC 17760C[null]2320[null]0
89227.525000male[null]1[null]PC 17757C[null][null]0[null]0
9050.495800male[null]131.000PC 17590SA24[null]0Trenton, NJ0
9150.000000male[null]1[null]113767SA32[null]0Seattle, WA0
9240.125000male[null]136.00013049CA10[null]0Winnipeg, MB0
9359.400000male[null]155.000PC 17603C[null][null]0New York, NY1
9426.550000male[null]133.000113790S[null]1090London0
95262.375000male[null]161.000PC 17608CB57 B59 B63 B66[null]3Haverford, PA / Cooperstown, NY1
9655.900000male[null]150.00013507SE44[null]0Duluth, MN1
9726.550000male[null]156.000113792S[null][null]0New York, NY0
9830.695800male[null]156.00017764CA7[null]0St James, Long Island, NY0
9960.000000male[null]124.00013695SC31[null]0Huntington, WV1
10026.000000male[null]1[null]113056SA19[null]0Streatham, Surrey0
Out[3]:
Rows: 1-100 of 1234 | Columns: 14
In [20]:
# using the dtype parameter
pandas_to_vertica(df = df, 
                  name = "titanic_pandas", 
                  schema = "public",
                  dtype = {"pclass": "Integer", 
                           "survived": "Integer", 
                           "name": "Varchar(164)", 
                           "sex": "Varchar(20)", 
                           "age": "Numeric(6,3)", 
                           "sibsp": "Integer", 
                           "parch": "Integer", 
                           "ticket": "Varchar(36)", 
                           "fare": "Numeric(10,5)", 
                           "cabin": "Varchar(30)", 
                           "embarked": "Varchar(20)", 
                           "boat": "Varchar(100)", 
                           "body": "Integer", 
                           "home.dest": "Varchar(100)"})
The table "public"."titanic_pandas" has been successfully created.
Out[20]:
123
pclass
Int
123
survived
Int
Abc
Varchar(164)
Abc
sex
Varchar(20)
123
age
Numeric(6,3)
123
sibsp
Int
123
parch
Int
Abc
ticket
Varchar(36)
123
fare
Numeric(10,5)
Abc
cabin
Varchar(30)
Abc
embarked
Varchar(20)
Abc
boat
Varchar(100)
123
body
Int
Abc
Varchar(100)
110female2.012113781151.55C22 C26S[null][null]
210female25.012113781151.55C22 C26S[null][null]
310male39.0001120500.0A36S[null][null]
410male[null]00PC 1731825.925[null]S[null][null]
510male24.001PC 17558247.5208B58 B60C[null][null]
610male36.0001305075.2417C6CA[null]
710male45.00011378435.5TS[null][null]
810male42.00011048926.55D22S[null][null]
910male41.00011305430.5A21S[null][null]
1010male[null]0011237939.6[null]C[null][null]
1110male45.00011305026.55B38S[null][null]
1210male[null]0011379831.0[null]S[null][null]
1310male33.0006955.0B51 B53 B55S[null][null]
1410male28.00011305947.1[null]S[null][null]
1510male17.00011305947.1[null]S[null][null]
1610male49.0001992426.0[null]S[null][null]
1710male46.010W.E.P. 573461.175E31S[null][null]
1810male[null]001120510.0[null]S[null][null]
1910male27.01013508136.7792C89C[null][null]
2010male[null]0011046552.0A14S[null][null]
2110male47.000572725.5875E58S[null][null]
2210male37.011PC 1775683.1583E52C[null][null]
2310male[null]0011379126.55[null]S[null][null]
2410male39.010PC 1759971.2833C85C[null][null]
2510male31.010F.C. 1275052.0B71S[null][null]
2610female36.000PC 1753131.6792A29C[null][null]
2710male[null]00PC 17483221.7792C95S[null][null]
2810male30.00011305127.75C111C[null][null]
2910male19.03219950263.0C23 C25 C27S[null][null]
3010male64.01419950263.0C23 C25 C27S[null][null]
3110male[null]0011377826.55D34S[null][null]
3210male[null]001120580.0B102S[null][null]
3310male37.01011380353.1C123S[null][null]
3410male24.000PC 1759379.2B86C[null][null]
3510male71.000PC 1775434.6542A5C[null][null]
3610male46.000PC 1759379.2B82 B84C[null][null]
3710male[null]0011379642.4[null]S[null][null]
3810male45.0103697383.475C83S[null][null]
3910male42.00011303842.5B11S[null][null]
4010male[null]001746351.8625E46S[null][null]
4110male55.00068050.0C39S[null][null]
4210male[null]00PC 1760030.6958[null]C14[null]
4310female50.000PC 1759528.7125C49C[null][null]
4410male50.00011304426.0E60S[null][null]
4510male41.0101746451.8625D21S[null][null]
4610male[null]0011302826.55C124S[null][null]
4710male[null]00PC 1761227.7208[null]C[null][null]
4810male30.00011380145.5[null]S[null][null]
4910male30.00011046926.0C106S[null][null]
5010male19.01011377353.1D30S[null][null]
5110male28.010PC 1760482.1708[null]C[null][null]
5210male55.00011378730.5C30S[null][null]
5310male47.00011379642.4[null]S[null][null]
5410male37.001PC 1759629.7C118C[null][null]
5510male[null]001120520.0[null]S[null][null]
5610male23.0001274993.5B24S[null][null]
5710male29.01011377666.6C2S[null][null]
5810male18.010PC 17758108.9C65C[null][null]
5910male38.000199720.0[null]S[null][null]
6010male[null]00PC 17757227.525[null]C[null][null]
6110male31.000PC 1759050.4958A24S[null][null]
6210male[null]0011376750.0A32S[null][null]
6310male36.0001304940.125A10C[null][null]
6410male55.010PC 1760359.4[null]C[null][null]
6510male61.013PC 17608262.375B57 B59 B63 B66C[null][null]
6610male50.0101350755.9E44S[null][null]
6710male56.00011379226.55[null]S[null][null]
6810male56.0001776430.6958A7C[null][null]
6910male24.0101369560.0C31S[null][null]
7010male[null]0011305626.0A19S[null][null]
7110male57.010PC 17569146.5208B78C[null][null]
7210male62.00011351426.55C87S[null][null]
7310male[null]00PC 1760527.7208[null]C[null][null]
7410female63.010PC 17483221.7792C55 C57S[null][null]
7510male52.01111041379.65E67S[null][null]
7610male49.01117421110.8833C68C[null][null]
7710male40.000PC 1760127.7208[null]C[null][null]
7810male47.0003696734.0208D46S[null][null]
7910male64.01011081375.25D37C[null][null]
8010male60.00011380026.55[null]S[null][null]
8110male54.0013528177.2875D26S[null][null]
8210male57.01136928164.8667[null]S[null][null]
8310male50.011113503211.5C80C[null][null]
8410male27.002113503211.5C82C[null][null]
8510male51.001PC 1759761.3792[null]C[null][null]
8610male[null]0011351035.0C128S[null][null]
8710male62.00011380726.55[null]S[null][null]
8811female29.00024160211.3375B5S2[null]
8911male0.9212113781151.55C22 C26S11[null]
9011male48.0001995226.55E12S3[null]
9111female63.0101350277.9583D7S10[null]
9211female53.0201176951.4792C101SD[null]
9311female18.010PC 17757227.525C62 C64C4[null]
9411female24.000PC 1747769.3B35C9[null]
9511male80.0002704230.0A23SB[null]
9611female50.001PC 17558247.5208B58 B60C6[null]
9711female32.0001181376.2917D15C8[null]
9811male37.0111175152.5542D35S5[null]
9911female47.0111175152.5542D35S5[null]
10011male26.00011136930.0C148C5[null]
Rows: 1-100 | Columns: 14

See Also

read_csv Ingests a CSV file into the Vertica DB.
read_json Ingests a JSON file into the Vertica DB.